def getEasyBuildVersion(rootdirectory):
# We make sure that the path given is totally expanded.
absrootdirectory = os.path.abspath(os.path.expandvars(rootdirectory))
# Appending temporarily to PYTHONPATH the EasyBuild directories and vsc-base
sys.path.insert(
0,
os.path.join(os.path.join(absrootdirectory, '.installRef'),
'easybuild-framework'))
sys.path.insert(
0,
os.path.join(os.path.join(absrootdirectory, '.installRef'),
'easybuild-easyblocks'))
sys.path.insert(
0,
os.path.join(os.path.join(absrootdirectory, '.installRef'),
'easybuild-easyconfigs'))
sys.path.insert(
0,
os.path.join(os.path.join(absrootdirectory, '.installRef'),
'vsc-base'))
# Importing the function that EasyBuild uses to determine its own version
from easybuild.tools.version import this_is_easybuild
# Getting the version of EasyBuild from the output message
msg = this_is_easybuild()
version = re.search("[0-9]*\.[0-9]*\.[0-9]*", msg).group(0)
# Removing from the PYTHONPATH the EasyBuild directories (cleanup)
sys.path.pop(0)
sys.path.pop(0)
sys.path.pop(0)
sys.path.pop(0)
return version
def log_start(eb_command_line, eb_tmpdir):
"""Log startup info."""
_log.info(this_is_easybuild())
# log used command line
_log.info("Command line: %s" % (' '.join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
def log_start(log, eb_command_line, eb_tmpdir):
"""Log startup info."""
log.info(this_is_easybuild())
# log used command line
log.info("Command line: %s", ' '.join(eb_command_line))
log.info("Using %s as temporary directory", eb_tmpdir)
class EasyBuildOptions(GeneralOption):
"""Easybuild generaloption class"""
VERSION = this_is_easybuild()
DEFAULT_LOGLEVEL = 'INFO'
DEFAULT_CONFIGFILES = get_default_configfiles()
ALLOPTSMANDATORY = False # allow more than one argument
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [
filetools.IGNORE, filetools.WARN, filetools.ERROR
]
try:
default_robot_path = get_paths_for("easyconfigs",
robot_path=None)[0]
except:
self.log.warning(
"basic_options: unable to determine default easyconfig path")
default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
"only-blocks":
("Only build listed blocks", None, "extend", None, "b", {
'metavar': "BLOCKS"
}),
"force":
(("Force to rebuild software even if it's already installed "
"(i.e. if it can be found as module)"), None, "store_true",
False, "f"),
"job": ("Submit the build as a job", None, "store_true", False),
"skip":
("Skip existing software (useful for installing additional packages)",
None, "store_true", False, "k"),
"robot":
("Path to search for easyconfigs for missing dependencies.", None,
"store_or_None", default_robot_path, "r", {
'metavar': "PATH"
}),
"stop": ("Stop the installation after certain step", "choice",
"store_or_None", "source", "s", all_stops),
"strict": ("Set strictness level", "choice", "store",
filetools.WARN, strictness_options),
"logtostdout": ("Redirect main log to stdout", None, "store_true",
False, "l"),
"dry-run": ("Resolve dependencies and print build list, then stop",
None, "store_true", False),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def software_options(self):
# software build options
descr = ("Software search and build options", (
"Specify software search and build options: EasyBuild will search for a "
"matching easyconfig and build it. When called with the try prefix "
"(i.e. --try-X ), EasyBuild will search for a matching easyconfig "
"and if none are found, try to generate one based on a close matching one "
"(NOTE: --try-X is best effort, it might produce wrong builds!)"))
opts = OrderedDict({
'software-name':
("Search and build software with name", None, 'store', None, {
'metavar': 'NAME'
}),
'software-version':
("Search and build software with version", None, 'store', None, {
'metavar': 'VERSION'
}),
'toolchain': ("Search and build with toolchain (name and version)",
None, 'extend', None, {
'metavar': 'NAME,VERSION'
}),
'toolchain-name':
("Search and build with toolchain name", None, 'store', None, {
'metavar': 'NAME'
}),
'toolchain-version': ("Search and build with toolchain version",
None, 'store', None, {
'metavar': 'VERSION'
}),
'amend':
(("Specify additional search and build parameters (can be used multiple times); "
"for example: versionprefix=foo or patches=one.patch,two.patch)"
), None, 'append', None, {
'metavar': 'VAR=VALUE[,VALUE]'
}),
})
longopts = opts.keys()
for longopt in longopts:
hlp = opts[longopt][0]
hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:])
opts["try-%s" % longopt] = (hlp, ) + opts[longopt][1:]
self.log.debug("software_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
opts = {
"easyblock": (
"easyblock to use for processing the spec file or dumping the options",
None,
"store",
None,
"e",
{
'metavar': "CLASS"
},
),
"pretend": (
("Does the build/installation in "
"a test directory located in $HOME/easybuildinstall "),
None,
"store_true",
False,
"p",
),
"skip-test-cases": (
"Skip running test cases",
None,
"store_true",
False,
"t",
),
}
self.log.debug("override_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def config_options(self):
# config options
descr = ("Configuration options", "Configure EasyBuild behavior.")
oldstyle_defaults = get_default_oldstyle_configfile_defaults()
opts = {
"config": (
"Path to EasyBuild config file",
None,
'store',
oldstyle_defaults['config'],
"C",
),
'prefix':
(('Change prefix for buildpath, installpath, sourcepath and repositorypath '
'(repositorypath prefix is only relevant in case of FileRepository repository)'
'(used prefix for defaults %s)' % oldstyle_defaults['prefix']),
None, 'store', None),
'buildpath': ('Temporary build path', None, 'store',
oldstyle_defaults['buildpath']),
'installpath': ('Final install path', None, 'store',
oldstyle_defaults['installpath']),
'subdir-modules': ('Subdir in installpath for modules', None,
'store', oldstyle_defaults['subdir_modules']),
'subdir-software': ('Subdir in installpath for software', None,
'store', oldstyle_defaults['subdir_software']),
'repository': ('Repository type, using repositorypath', 'choice',
'store', oldstyle_defaults['repository'],
sorted(avail_repositories().keys())),
'repositorypath':
(('Repository path, used by repository '
'(is passed as list of arguments to create the repository instance). '
'For more info, use --avail-repositories.'), 'strlist', 'store',
oldstyle_defaults['repositorypath'][
oldstyle_defaults['repository']]),
"avail-repositories": (
"Show all repository types (incl. non-usable)",
None,
"store_true",
False,
),
'logfile-format':
('Directory name and format of the log file ', 'strtuple', 'store',
oldstyle_defaults['logfile_format'], {
'metavar': 'DIR,FORMAT'
}),
'tmp-logdir':
('Log directory where temporary log files are stored', None,
'store', oldstyle_defaults['tmp_logdir']),
'sourcepath':
('Path(s) to where sources should be downloaded (string, colon-separated)',
None, 'store', oldstyle_defaults['sourcepath']),
'moduleclasses':
(('Extend supported module classes'
' (For more info on the default classes, use --show-default-moduleclasses)'
), None, 'extend', oldstyle_defaults['moduleclasses']),
'show-default-moduleclasses':
('Show default module classes with description', None,
'store_true', False),
'modules-tool': ('Modules tool to use', 'choice', 'store',
oldstyle_defaults['modules_tool'],
sorted(avail_modules_tools().keys())),
"avail-modules-tools": (
"Show all supported module tools",
None,
"store_true",
False,
),
'module-naming-scheme':
('Module naming scheme', 'choice', 'store',
oldstyle_defaults['module_naming_scheme'],
sorted(avail_module_naming_schemes().keys())),
"avail-module-naming-schemes": (
"Show all supported module naming schemes",
None,
"store_true",
False,
),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
"testoutput":
("Path to where a job should place the output (to be set within jobscript)",
None, "store", None),
}
self.log.debug("config_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def informative_options(self):
# informative options
descr = ("Informative options", "Obtain information about EasyBuild.")
opts = {
"avail-easyconfig-params": (
("Show all easyconfig parameters (include "
"easyblock-specific ones by using -e)"),
None,
"store_true",
False,
"a",
),
"avail-easyconfig-templates":
(("Show all template names and template constants "
"that can be used in easyconfigs."), None, "store_true", False),
"avail-easyconfig-constants":
(("Show all constants that can be used in easyconfigs."), None,
"store_true", False),
"avail-easyconfig-licenses":
(("Show all license constants that can be used in easyconfigs."),
None, "store_true", False),
"list-easyblocks":
("Show list of available easyblocks", "choice", "store_or_None",
"simple", ["simple", "detailed"]),
"list-toolchains": ("Show list of known toolchains", None,
"store_true", False),
"search": ("Search for easyconfig files in the robot directory",
None, "store", None, {
'metavar': "STR"
}),
"dep-graph": (
"Create dependency graph",
None,
"store",
None,
{
'metavar': "depgraph.<ext>"
},
),
}
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def regtest_options(self):
# regression test options
descr = ("Regression test options",
"Run and control an EasyBuild regression test.")
opts = {
"regtest":
("Enable regression test mode", None, "store_true", False),
"regtest-online": (
"Enable online regression test mode",
None,
"store_true",
False,
),
"sequential": (
"Specify this option if you want to prevent parallel build",
None,
"store_true",
False,
),
"regtest-output-dir": (
"Set output directory for test-run",
None,
"store",
None,
{
'metavar': "DIR"
},
),
"aggregate-regtest": (
"Collect all the xmls inside the given directory and generate a single file",
None,
"store",
None,
{
'metavar': "DIR"
},
),
}
self.log.debug("regtest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def easyconfig_options(self):
# easyconfig options (to be passed to easyconfig instance)
descr = ("Options for Easyconfigs",
"Options to be passed to all Easyconfig.")
opts = None
self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyconfig')
def easyblock_options(self):
# easyblock options (to be passed to easyblock instance)
descr = ("Options for Easyblocks",
"Options to be passed to all Easyblocks.")
opts = None
self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyblock')
def unittest_options(self):
# unittest options
descr = ("Unittest options",
"Options dedicated to unittesting (experts only).")
opts = {
"file": ("Log to this file in unittest mode", None, "store", None),
}
self.log.debug("unittest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='unittest')
def validate(self):
"""Additional validation of options"""
stop_msg = []
if self.options.toolchain and not len(self.options.toolchain) == 2:
stop_msg.append('--toolchain requires NAME,VERSION (given %s)' %
(','.join(self.options.toolchain)))
if self.options.try_toolchain and not len(
self.options.try_toolchain) == 2:
stop_msg.append(
'--try-toolchain requires NAME,VERSION (given %s)' %
(','.join(self.options.try_toolchain)))
if len(stop_msg) > 0:
indent = " " * 2
stop_msg = ['%s%s' % (indent, x) for x in stop_msg]
stop_msg.insert(
0, 'ERROR: Found %s problems validating the options:' %
len(stop_msg))
print "\n".join(stop_msg)
sys.exit(1)
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict(
) # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def _postprocess_config(self):
"""Postprocessing of configuration options"""
if self.options.prefix is not None:
changed_defaults = get_default_oldstyle_configfile_defaults(
self.options.prefix)
for dest in [
'installpath', 'buildpath', 'sourcepath', 'repositorypath'
]:
if not self.options._action_taken.get(dest, False):
new_def = changed_defaults[dest]
if dest == 'repositorypath':
setattr(self.options, dest,
new_def[changed_defaults['repository']])
else:
setattr(self.options, dest, new_def)
# LEGACY this line is here for oldstyle reasons
self.log.deprecated(
'Fake action taken to distinguish from default', '2.0')
self.options._action_taken[dest] = True
if self.options.pretend:
self.options.installpath = get_pretend_installpath()
def _postprocess_list_avail(self):
"""Create all the additional info that can be requested (exit at the end)"""
msg = ''
# dump possible easyconfig params
if self.options.avail_easyconfig_params:
msg += self.avail_easyconfig_params()
# dump easyconfig template options
if self.options.avail_easyconfig_templates:
msg += template_documentation()
# dump easyconfig constant options
if self.options.avail_easyconfig_constants:
msg += constant_documentation()
# dump easyconfig license options
if self.options.avail_easyconfig_licenses:
msg += license_documentation()
# dump available easyblocks
if self.options.list_easyblocks:
msg += self.avail_easyblocks()
# dump known toolchains
if self.options.list_toolchains:
msg += self.avail_toolchains()
# dump known repository types
if self.options.avail_repositories:
msg += self.avail_repositories()
# dump supported modules tools
if self.options.avail_modules_tools:
msg += self.avail_list('modules tools', avail_modules_tools())
# dump supported module naming schemes
if self.options.avail_module_naming_schemes:
msg += self.avail_list('module naming schemes',
avail_module_naming_schemes())
# dump default moduleclasses with description
if self.options.show_default_moduleclasses:
msg += self.show_default_moduleclasses()
if self.options.unittest_file:
self.log.info(msg)
else:
print msg
sys.exit(0)
def avail_easyconfig_params(self):
"""
Print the available easyconfig parameters, for the given easyblock.
"""
app = get_class(self.options.easyblock)
extra = app.extra_options()
mapping = convert_to_help(extra, has_default=False)
if len(extra) > 0:
ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__
extra_names = [x[0] for x in extra]
else:
ebb_msg = ''
extra_names = []
txt = ["Available easyconfig parameters%s" % ebb_msg]
params = [(k, v) for (k, v) in mapping.items()
if k.upper() not in ['HIDDEN']]
for key, values in params:
txt.append("%s" % key.upper())
txt.append('-' * len(key))
for name, value in values:
tabs = "\t" * (3 - (len(name) + 1) / 8)
if name in extra_names:
starred = '(*)'
else:
starred = ''
txt.append("%s%s:%s%s" % (name, starred, tabs, value))
txt.append('')
return "\n".join(txt)
def avail_classes_tree(self, classes, classNames, detailed, depth=0):
"""Print list of classes as a tree."""
txt = []
for className in classNames:
classInfo = classes[className]
if detailed:
txt.append("%s|-- %s (%s)" %
("| " * depth, className, classInfo['module']))
else:
txt.append("%s|-- %s" % ("| " * depth, className))
if 'children' in classInfo:
txt.extend(
self.avail_classes_tree(classes, classInfo['children'],
detailed, depth + 1))
return txt
def avail_easyblocks(self):
"""Get a class tree for easyblocks."""
detailed = self.options.list_easyblocks == "detailed"
module_regexp = re.compile(r"^([^_].*)\.py$")
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
for package in [
"easybuild.easyblocks", "easybuild.easyblocks.generic"
]:
__import__(package)
# determine paths for this package
paths = sys.modules[package].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for f in os.listdir(path):
res = module_regexp.match(f)
if res:
__import__("%s.%s" % (package, res.group(1)))
def add_class(classes, cls):
"""Add a new class, and all of its subclasses."""
children = cls.__subclasses__()
classes.update({
cls.__name__: {
'module': cls.__module__,
'children': [x.__name__ for x in children]
}
})
for child in children:
add_class(classes, child)
roots = [EasyBlock, Extension]
classes = {}
for root in roots:
add_class(classes, root)
# Print the tree, start with the roots
txt = []
for root in roots:
root = root.__name__
if detailed:
txt.append("%s (%s)" % (root, classes[root]['module']))
else:
txt.append("%s" % root)
if 'children' in classes[root]:
txt.extend(
self.avail_classes_tree(classes, classes[root]['children'],
detailed))
txt.append("")
return "\n".join(txt)
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(
version='1.2.3'
) # version doesn't matter here, but something needs to be there
tc_elems = set([
y for x in dir(tc) if x.endswith('_MODULE_NAME')
for y in eval("tc.%s" % x)
])
txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems))))
return '\n'.join(txt)
def avail_repositories(self):
"""Show list of known repository types."""
repopath_defaults = get_default_oldstyle_configfile_defaults(
)['repositorypath']
all_repos = avail_repositories(check_useable=False)
usable_repos = avail_repositories(check_useable=True).keys()
indent = ' ' * 2
txt = ['All avaialble repository types']
repos = sorted(all_repos.keys())
for repo in repos:
if repo in usable_repos:
missing = ''
else:
missing = ' (*Not usable*, something is missing (eg a specific module))'
if repo in repopath_defaults:
default = ' (Default arguments: %s)' % (
repopath_defaults[repo])
else:
default = ' (No default arguments)'
txt.append("%s%s%s%s" % (indent, repo, default, missing))
txt.append("%s%s" % (indent * 2, all_repos[repo].DESCRIPTION))
return "\n".join(txt)
def avail_list(self, name, items):
"""Show list of available values passed by argument."""
return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items))
def show_default_moduleclasses(self):
"""Show list of default moduleclasses and description."""
txt = ["Default available moduleclasses"]
indent = " " * 2
maxlen = max([len(x[0])
for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space
for name, descr in DEFAULT_MODULECLASSES:
txt.append("%s%s:%s%s" % (indent, name,
(" " * (maxlen - len(name))), descr))
return "\n".join(txt)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# specified robot paths are preferred over installed easyconfig files
if robot_path:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'group': options.group,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_online': options.regtest_online,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for f in files:
if not generated and try_to_generate and build_specs:
ec_file = tweak(f, None, build_specs)
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n"
)
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-")
os.close(fd)
fancylogger.logToFile(logfile)
print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section("config"))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key("name"):
paths = [
obtain_path(
software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing
)
]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(
(
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"
),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing,
)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info(
"Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])
)
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
class EasyBuildOptions(GeneralOption):
"""Easybuild generaloption class"""
VERSION = this_is_easybuild()
DEFAULT_LOGLEVEL = 'INFO'
DEFAULT_CONFIGFILES = get_default_configfiles()
ALLOPTSMANDATORY = False # allow more than one argument
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [run.IGNORE, run.WARN, run.ERROR]
try:
default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0]
except:
self.log.warning("basic_options: unable to determine default easyconfig path")
default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False),
'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'),
'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)",
None, 'store_true', False, 'f'),
'job': ("Submit the build as a job", None, 'store_true', False),
'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'),
'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', {'metavar': 'BLOCKS'}),
'robot': ("Path(s) to search for easyconfigs for missing dependencies (colon-separated)" ,
None, 'store_or_None', default_robot_path, 'r', {'metavar': 'PATH'}),
'skip': ("Skip existing software (useful for installing additional packages)",
None, 'store_true', False, 'k'),
'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops),
'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def software_options(self):
# software build options
descr = ("Software search and build options",
("Specify software search and build options: EasyBuild will search for a "
"matching easyconfig and build it. When called with the try prefix "
"(i.e. --try-X ), EasyBuild will search for a matching easyconfig "
"and if none are found, try to generate one based on a close matching one "
"(NOTE: --try-X is best effort, it might produce wrong builds!)")
)
opts = OrderedDict({
'amend':(("Specify additional search and build parameters (can be used multiple times); "
"for example: versionprefix=foo or patches=one.patch,two.patch)"),
None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}),
'software-name': ("Search and build software with name",
None, 'store', None, {'metavar': 'NAME'}),
'software-version': ("Search and build software with version",
None, 'store', None, {'metavar': 'VERSION'}),
'toolchain': ("Search and build with toolchain (name and version)",
None, 'extend', None, {'metavar': 'NAME,VERSION'}),
'toolchain-name': ("Search and build with toolchain name",
None, 'store', None, {'metavar': 'NAME'}),
'toolchain-version': ("Search and build with toolchain version",
None, 'store', None, {'metavar': 'VERSION'}),
})
longopts = opts.keys()
for longopt in longopts:
hlp = opts[longopt][0]
hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:])
opts["try-%s" % longopt] = (hlp,) + opts[longopt][1:]
# additional options that don't need a --try equivalent
opts.update({
'from-pr': ("Obtain easyconfigs from specified PR", int, 'store', None, {'metavar': 'PR#'}),
})
self.log.debug("software_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
opts = OrderedDict({
'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function",
None, 'store_true', False),
'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True),
'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.",
None, 'store', None),
'easyblock': ("easyblock to use for processing the spec file or dumping the options",
None, 'store', None, 'e', {'metavar': 'CLASS'}),
'experimental': ("Allow experimental code (with behaviour that can be changed or removed at any given time).",
None, 'store_true', False),
'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),
'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
'oldstyleconfig': ("Look for and use the oldstyle configuration file.",
None, 'store_true', True),
'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
None, 'store_true', False, 'p'),
'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False),
'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
None, 'store', None),
})
self.log.debug("override_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def config_options(self):
# config options
descr = ("Configuration options", "Configure EasyBuild behavior.")
oldstyle_defaults = get_default_oldstyle_configfile_defaults()
opts = OrderedDict({
'avail-module-naming-schemes': ("Show all supported module naming schemes",
None, 'store_true', False,),
'avail-modules-tools': ("Show all supported module tools",
None, "store_true", False,),
'avail-repositories': ("Show all repository types (incl. non-usable)",
None, "store_true", False,),
'buildpath': ("Temporary build path", None, 'store', oldstyle_defaults['buildpath']),
'ignore-dirs': ("Directory names to ignore when searching for files/dirs",
'strlist', 'store', ['.git', '.svn']),
'installpath': ("Install path for software and modules", None, 'store', oldstyle_defaults['installpath']),
'config': ("Path to EasyBuild config file",
None, 'store', oldstyle_defaults['config'], 'C'),
'logfile-format': ("Directory name and format of the log file",
'strtuple', 'store', oldstyle_defaults['logfile_format'], {'metavar': 'DIR,FORMAT'}),
'module-naming-scheme': ("Module naming scheme",
'choice', 'store', oldstyle_defaults['module_naming_scheme'],
sorted(avail_module_naming_schemes().keys())),
'moduleclasses': (("Extend supported module classes "
"(For more info on the default classes, use --show-default-moduleclasses)"),
None, 'extend', oldstyle_defaults['moduleclasses']),
'modules-footer': ("Path to file containing footer to be added to all generated module files",
None, 'store_or_None', None, {'metavar': "PATH"}),
'modules-tool': ("Modules tool to use",
'choice', 'store', oldstyle_defaults['modules_tool'],
sorted(avail_modules_tools().keys())),
'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath "
"(repositorypath prefix is only relevant in case of FileRepository repository) "
"(used prefix for defaults %s)" % oldstyle_defaults['prefix']),
None, 'store', None),
'recursive-module-unload': ("Enable generating of modules that unload recursively.",
None, 'store_true', False),
'repository': ("Repository type, using repositorypath",
'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())),
'repositorypath': (("Repository path, used by repository "
"(is passed as list of arguments to create the repository instance). "
"For more info, use --avail-repositories."),
'strlist', 'store',
oldstyle_defaults['repositorypath'][oldstyle_defaults['repository']]),
'show-default-moduleclasses': ("Show default module classes with description",
None, 'store_true', False),
'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)",
None, 'store', oldstyle_defaults['sourcepath']),
'subdir-modules': ("Installpath subdir for modules", None, 'store', oldstyle_defaults['subdir_modules']),
'subdir-software': ("Installpath subdir for software", None, 'store', oldstyle_defaults['subdir_software']),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
'testoutput': ("Path to where a job should place the output (to be set within jobscript)",
None, 'store', None),
'tmp-logdir': ("Log directory where temporary log files are stored",
None, 'store', oldstyle_defaults['tmp_logdir']),
'tmpdir': ('Directory to use for temporary storage', None, 'store', None),
})
self.log.debug("config_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def informative_options(self):
# informative options
descr = ("Informative options", "Obtain information about EasyBuild.")
opts = OrderedDict({
'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs",
None, 'store_true', False),
'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs",
None, 'store_true', False),
'avail-easyconfig-params': (("Show all easyconfig parameters (include "
"easyblock-specific ones by using -e)"),
None, "store_true", False, 'a'),
'avail-easyconfig-templates': (("Show all template names and template constants "
"that can be used in easyconfigs"),
None, 'store_true', False),
'dep-graph': ("Create dependency graph",
None, "store", None, {'metavar': 'depgraph.<ext>'}),
'list-easyblocks': ("Show list of available easyblocks",
'choice', 'store_or_None', 'simple', ['simple', 'detailed']),
'list-toolchains': ("Show list of known toolchains",
None, 'store_true', False),
'search': ("Search for easyconfig files in the robot directory, print full paths",
None, 'store', None, {'metavar': 'STR'}),
'search-short': ("Search for easyconfig files in the robot directory, print short paths",
None, 'store', None, 'S', {'metavar': 'STR'}),
})
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def regtest_options(self):
# regression test options
descr = ("Regression test options", "Run and control an EasyBuild regression test.")
opts = OrderedDict({
'aggregate-regtest': ("Collect all the xmls inside the given directory and generate a single file",
None, 'store', None, {'metavar': 'DIR'}),
'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'),
'github-user': ("GitHub username", None, 'store', None),
'regtest': ("Enable regression test mode",
None, 'store_true', False),
'regtest-output-dir': ("Set output directory for test-run",
None, 'store', None, {'metavar': 'DIR'}),
'sequential': ("Specify this option if you want to prevent parallel build",
None, 'store_true', False),
'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', None),
})
self.log.debug("regtest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def easyconfig_options(self):
# easyconfig options (to be passed to easyconfig instance)
descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.")
opts = None
self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyconfig')
def easyblock_options(self):
# easyblock options (to be passed to easyblock instance)
descr = ("Options for Easyblocks", "Options to be passed to all Easyblocks.")
opts = None
self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyblock')
def unittest_options(self):
# unittest options
descr = ("Unittest options", "Options dedicated to unittesting (experts only).")
opts = OrderedDict({
'file': ("Log to this file in unittest mode", None, 'store', None),
})
self.log.debug("unittest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='unittest')
def validate(self):
"""Additional validation of options"""
stop_msg = []
if self.options.toolchain and not len(self.options.toolchain) == 2:
stop_msg.append('--toolchain requires NAME,VERSION (given %s)' %
(','.join(self.options.toolchain)))
if self.options.try_toolchain and not len(self.options.try_toolchain) == 2:
stop_msg.append('--try-toolchain requires NAME,VERSION (given %s)' %
(','.join(self.options.try_toolchain)))
if self.options.umask:
umask_regex = re.compile('^[0-7]{3}$')
if not umask_regex.match(self.options.umask):
stop_msg.append("--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern)
if len(stop_msg) > 0:
indent = " "*2
stop_msg = ['%s%s' % (indent, x) for x in stop_msg]
stop_msg.insert(0, 'ERROR: Found %s problems validating the options:' % len(stop_msg))
print "\n".join(stop_msg)
sys.exit(1)
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
self.log.error("Required support for using GitHub API is not available (see warnings).")
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.")
if self.options.github_user is None:
self.log.error("No GitHub user name provided, required for fetching GitHub token.")
token = fetch_github_token(self.options.github_user)
if token is None:
self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user)
self._postprocess_config()
def _postprocess_config(self):
"""Postprocessing of configuration options"""
if self.options.prefix is not None:
changed_defaults = get_default_oldstyle_configfile_defaults(self.options.prefix)
for dest in ['installpath', 'buildpath', 'sourcepath', 'repositorypath']:
if not self.options._action_taken.get(dest, False):
new_def = changed_defaults[dest]
if dest == 'repositorypath':
setattr(self.options, dest, new_def[changed_defaults['repository']])
else:
setattr(self.options, dest, new_def)
# LEGACY this line is here for oldstyle reasons
self.log.deprecated('Fake action taken to distinguish from default', '2.0')
self.options._action_taken[dest] = True
if self.options.pretend:
self.options.installpath = get_pretend_installpath()
# split supplied list of robot paths to obtain a list
if self.options.robot:
class RobotPath(ListOfStrings):
SEPARATOR_LIST = os.pathsep
# explicit definition of __str__ is required for unknown reason related to the way Wrapper is defined
__str__ = ListOfStrings.__str__
self.options.robot = RobotPath(self.options.robot)
def _postprocess_list_avail(self):
"""Create all the additional info that can be requested (exit at the end)"""
msg = ''
# dump possible easyconfig params
if self.options.avail_easyconfig_params:
msg += self.avail_easyconfig_params()
# dump easyconfig template options
if self.options.avail_easyconfig_templates:
msg += template_documentation()
# dump easyconfig constant options
if self.options.avail_easyconfig_constants:
msg += constant_documentation()
# dump easyconfig license options
if self.options.avail_easyconfig_licenses:
msg += license_documentation()
# dump available easyblocks
if self.options.list_easyblocks:
msg += self.avail_easyblocks()
# dump known toolchains
if self.options.list_toolchains:
msg += self.avail_toolchains()
# dump known repository types
if self.options.avail_repositories:
msg += self.avail_repositories()
# dump supported modules tools
if self.options.avail_modules_tools:
msg += self.avail_list('modules tools', avail_modules_tools())
# dump supported module naming schemes
if self.options.avail_module_naming_schemes:
msg += self.avail_list('module naming schemes', avail_module_naming_schemes())
# dump default moduleclasses with description
if self.options.show_default_moduleclasses:
msg += self.show_default_moduleclasses()
if self.options.unittest_file:
self.log.info(msg)
else:
print msg
sys.exit(0)
def avail_easyconfig_params(self):
"""
Print the available easyconfig parameters, for the given easyblock.
"""
app = get_easyblock_class(self.options.easyblock)
extra = app.extra_options()
mapping = convert_to_help(extra, has_default=False)
if len(extra) > 0:
ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__
extra_names = [x[0] for x in extra]
else:
ebb_msg = ''
extra_names = []
txt = ["Available easyconfig parameters%s" % ebb_msg]
params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']]
for key, values in params:
txt.append("%s" % key.upper())
txt.append('-' * len(key))
for name, value in values:
tabs = "\t" * (3 - (len(name) + 1) / 8)
if name in extra_names:
starred = '(*)'
else:
starred = ''
txt.append("%s%s:%s%s" % (name, starred, tabs, value))
txt.append('')
return "\n".join(txt)
def avail_classes_tree(self, classes, classNames, detailed, depth=0):
"""Print list of classes as a tree."""
txt = []
for className in classNames:
classInfo = classes[className]
if detailed:
txt.append("%s|-- %s (%s)" % ("| " * depth, className, classInfo['module']))
else:
txt.append("%s|-- %s" % ("| " * depth, className))
if 'children' in classInfo:
txt.extend(self.avail_classes_tree(classes, classInfo['children'], detailed, depth + 1))
return txt
def avail_easyblocks(self):
"""Get a class tree for easyblocks."""
detailed = self.options.list_easyblocks == "detailed"
module_regexp = re.compile(r"^([^_].*)\.py$")
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
for package in ["easybuild.easyblocks", "easybuild.easyblocks.generic"]:
__import__(package)
# determine paths for this package
paths = sys.modules[package].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for f in os.listdir(path):
res = module_regexp.match(f)
if res:
__import__("%s.%s" % (package, res.group(1)))
def add_class(classes, cls):
"""Add a new class, and all of its subclasses."""
children = cls.__subclasses__()
classes.update({cls.__name__: {
'module': cls.__module__,
'children': [x.__name__ for x in children]
}
})
for child in children:
add_class(classes, child)
roots = [EasyBlock, Extension]
classes = {}
for root in roots:
add_class(classes, root)
# Print the tree, start with the roots
txt = []
for root in roots:
root = root.__name__
if detailed:
txt.append("%s (%s)" % (root, classes[root]['module']))
else:
txt.append("%s" % root)
if 'children' in classes[root]:
txt.extend(self.avail_classes_tree(classes, classes[root]['children'], detailed))
txt.append("")
return "\n".join(txt)
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there
tc_elems = set([y for x in dir(tc) if x.endswith('_MODULE_NAME') for y in eval("tc.%s" % x)])
txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems))))
return '\n'.join(txt)
def avail_repositories(self):
"""Show list of known repository types."""
repopath_defaults = get_default_oldstyle_configfile_defaults()['repositorypath']
all_repos = avail_repositories(check_useable=False)
usable_repos = avail_repositories(check_useable=True).keys()
indent = ' ' * 2
txt = ['All avaliable repository types']
repos = sorted(all_repos.keys())
for repo in repos:
if repo in usable_repos:
missing = ''
else:
missing = ' (*not usable*, something is missing (e.g. a required Python module))'
if repo in repopath_defaults:
default = ' (default arguments: %s)' % ', '.join(repopath_defaults[repo])
else:
default = ' (no default arguments)'
txt.append("%s* %s%s%s" % (indent, repo, default, missing))
txt.append("%s%s" % (indent * 3, all_repos[repo].DESCRIPTION))
return "\n".join(txt)
def avail_list(self, name, items):
"""Show list of available values passed by argument."""
return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items))
def show_default_moduleclasses(self):
"""Show list of default moduleclasses and description."""
txt = ["Default available moduleclasses"]
indent = " " * 2
maxlen = max([len(x[0]) for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space
for name, descr in DEFAULT_MODULECLASSES:
txt.append("%s%s:%s%s" % (indent, name, (" " * (maxlen - len(name))), descr))
return "\n".join(txt)
class EasyBuildOptions(GeneralOption):
"""Easybuild generaloption class"""
VERSION = this_is_easybuild()
DEFAULT_LOGLEVEL = 'INFO'
DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES + [DEFAULT_USER_CFGFILE]
ALLOPTSMANDATORY = False # allow more than one argument
def __init__(self, *args, **kwargs):
"""Constructor."""
self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR,
robot_path=None) or []
# set up constants to seed into config files parser, by section
self.go_cfg_constants = {
self.DEFAULTSECT: {
'DEFAULT_ROBOT_PATHS':
(os.pathsep.join(self.default_robot_paths),
"List of default robot paths ('%s'-separated)" % os.pathsep),
}
}
# update or define go_configfiles_initenv in named arguments to pass to parent constructor
go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {})
for section, constants in self.go_cfg_constants.items():
constants = dict([(name, value)
for (name, (value, _)) in constants.items()])
go_cfg_initenv.setdefault(section, {}).update(constants)
super(EasyBuildOptions, self).__init__(*args, **kwargs)
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [run.IGNORE, run.WARN, run.ERROR]
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
'dry-run': ("Print build overview incl. dependencies (full paths)",
None, 'store_true', False),
'dry-run-short':
("Print build overview incl. dependencies (short paths)", None,
'store_true', False, 'D'),
'force':
("Force to rebuild software even if it's already installed (i.e. if it can be found as module)",
None, 'store_true', False, 'f'),
'job': ("Submit the build as a job", None, 'store_true', False),
'logtostdout': ("Redirect main log to stdout", None, 'store_true',
False, 'l'),
'only-blocks': ("Only build listed blocks", None, 'extend', None,
'b', {
'metavar': 'BLOCKS'
}),
'robot':
("Enable dependency resolution, using easyconfigs in specified paths",
'pathlist', 'store_or_None', [], 'r', {
'metavar': 'PATH[%sPATH]' % os.pathsep
}),
'robot-paths':
("Additional paths to consider by robot for easyconfigs (--robot paths get priority)",
'pathlist', 'store', self.default_robot_paths, {
'metavar': 'PATH[%sPATH]' % os.pathsep
}),
'skip':
("Skip existing software (useful for installing additional packages)",
None, 'store_true', False, 'k'),
'stop': ("Stop the installation after certain step", 'choice',
'store_or_None', 'source', 's', all_stops),
'strict': ("Set strictness level", 'choice', 'store', run.WARN,
strictness_options),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def software_options(self):
# software build options
descr = ("Software search and build options", (
"Specify software search and build options: EasyBuild will search for a "
"matching easyconfig and build it. When called with the try prefix "
"(i.e. --try-X ), EasyBuild will search for a matching easyconfig "
"and if none are found, try to generate one based on a close matching one "
"(NOTE: --try-X is best effort, it might produce wrong builds!)"))
opts = OrderedDict({
'amend':
(("Specify additional search and build parameters (can be used multiple times); "
"for example: versionprefix=foo or patches=one.patch,two.patch)"
), None, 'append', None, {
'metavar': 'VAR=VALUE[,VALUE]'
}),
'software':
("Search and build software with given name and version", None,
'extend', None, {
'metavar': 'NAME,VERSION'
}),
'software-name': ("Search and build software with given name",
None, 'store', None, {
'metavar': 'NAME'
}),
'software-version':
("Search and build software with given version", None, 'store',
None, {
'metavar': 'VERSION'
}),
'toolchain':
("Search and build with given toolchain (name and version)", None,
'extend', None, {
'metavar': 'NAME,VERSION'
}),
'toolchain-name': ("Search and build with given toolchain name",
None, 'store', None, {
'metavar': 'NAME'
}),
'toolchain-version':
("Search and build with given toolchain version", None, 'store',
None, {
'metavar': 'VERSION'
}),
})
longopts = opts.keys()
for longopt in longopts:
hlp = opts[longopt][0]
hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:])
opts["try-%s" % longopt] = (hlp, ) + opts[longopt][1:]
# additional options that don't need a --try equivalent
opts.update({
'from-pr':
("Obtain easyconfigs from specified PR", int, 'store', None, {
'metavar': 'PR#'
}),
})
self.log.debug("software_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
opts = OrderedDict({
'allow-modules-tool-mismatch':
("Allow mismatch of modules tool and definition of 'module' function",
None, 'store_true', False),
'cleanup-builddir':
("Cleanup build dir after successful installation.", None,
'store_true', True),
'deprecated':
("Run pretending to be (future) version, to test removal of deprecated code.",
None, 'store', None),
'download-timeout':
("Timeout for initiating downloads (in seconds)", None, 'store',
None),
'easyblock':
("easyblock to use for processing the spec file or dumping the options",
None, 'store', None, 'e', {
'metavar': 'CLASS'
}),
'experimental':
("Allow experimental code (with behaviour that can be changed/removed at any given time).",
None, 'store_true', False),
'group':
("Group to be used for software installations (only verified, not set)",
None, 'store', None),
'hidden':
("Install 'hidden' module file(s) by prefixing their name with '.'",
None, 'store_true', False),
'ignore-osdeps': ("Ignore any listed OS dependencies", None,
'store_true', False),
'filter-deps':
("Comma separated list of dependencies that you DON'T want to install with EasyBuild, "
"because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)",
str, 'extend', None),
'oldstyleconfig':
("Look for and use the oldstyle configuration file.", None,
'store_true', True),
'optarch':
("Set architecture optimization, overriding native architecture optimizations",
None, 'store', None),
'pretend':
(("Does the build/installation in a test directory located in $HOME/easybuildinstall"
), None, 'store_true', False, 'p'),
'set-gid-bit': ("Set group ID bit on newly created directories",
None, 'store_true', False),
'sticky-bit': ("Set sticky bit on newly created directories", None,
'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true',
False, 't'),
'umask':
("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
None, 'store', None),
'update-modules-tool-cache':
("Update modules tool cache file(s) after generating module file",
None, 'store_true', False),
})
self.log.debug("override_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def config_options(self):
# config options
descr = ("Configuration options", "Configure EasyBuild behavior.")
opts = OrderedDict({
'avail-module-naming-schemes': (
"Show all supported module naming schemes",
None,
'store_true',
False,
),
'avail-modules-tools': (
"Show all supported module tools",
None,
"store_true",
False,
),
'avail-repositories': (
"Show all repository types (incl. non-usable)",
None,
"store_true",
False,
),
'buildpath': ("Temporary build path", None, 'store',
mk_full_default_path('buildpath')),
'ignore-dirs':
("Directory names to ignore when searching for files/dirs",
'strlist', 'store', ['.git', '.svn']),
'installpath': ("Install path for software and modules", None,
'store', mk_full_default_path('installpath')),
# purposely take a copy for the default logfile format
'logfile-format':
("Directory name and format of the log file", 'strtuple', 'store',
DEFAULT_LOGFILE_FORMAT[:], {
'metavar': 'DIR,FORMAT'
}),
'module-naming-scheme':
("Module naming scheme", 'choice', 'store', DEFAULT_MNS,
sorted(avail_module_naming_schemes().keys())),
'moduleclasses':
(("Extend supported module classes "
"(For more info on the default classes, use --show-default-moduleclasses)"
), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]),
'modules-footer':
("Path to file containing footer to be added to all generated module files",
None, 'store_or_None', None, {
'metavar': "PATH"
}),
'modules-tool': ("Modules tool to use", 'choice', 'store',
DEFAULT_MODULES_TOOL,
sorted(avail_modules_tools().keys())),
'prefix':
(("Change prefix for buildpath, installpath, sourcepath and repositorypath "
"(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store',
None),
'recursive-module-unload':
("Enable generating of modules that unload recursively.", None,
'store_true', False),
'repository': ("Repository type, using repositorypath", 'choice',
'store', DEFAULT_REPOSITORY,
sorted(avail_repositories().keys())),
'repositorypath':
(("Repository path, used by repository "
"(is passed as list of arguments to create the repository instance). "
"For more info, use --avail-repositories."), 'strlist', 'store',
[mk_full_default_path('repositorypath')]),
'show-default-moduleclasses':
("Show default module classes with description", None,
'store_true', False),
'sourcepath':
("Path(s) to where sources should be downloaded (string, colon-separated)",
None, 'store', mk_full_default_path('sourcepath')),
'subdir-modules': ("Installpath subdir for modules", None, 'store',
DEFAULT_PATH_SUBDIRS['subdir_modules']),
'subdir-software': ("Installpath subdir for software", None,
'store',
DEFAULT_PATH_SUBDIRS['subdir_software']),
'suffix-modules-path': ("Suffix for module files install path",
None, 'store', GENERAL_CLASS),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
'testoutput':
("Path to where a job should place the output (to be set within jobscript)",
None, 'store', None),
'tmp-logdir':
("Log directory where temporary log files are stored", None,
'store', None),
'tmpdir': ('Directory to use for temporary storage', None, 'store',
None),
})
self.log.debug("config_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def informative_options(self):
# informative options
descr = ("Informative options", "Obtain information about EasyBuild.")
opts = OrderedDict({
'avail-cfgfile-constants':
("Show all constants that can be used in configuration files",
None, 'store_true', False),
'avail-easyconfig-constants':
("Show all constants that can be used in easyconfigs", None,
'store_true', False),
'avail-easyconfig-licenses':
("Show all license constants that can be used in easyconfigs",
None, 'store_true', False),
'avail-easyconfig-params':
(("Show all easyconfig parameters (include "
"easyblock-specific ones by using -e)"), 'choice',
'store_or_None', FORMAT_TXT, [FORMAT_RST, FORMAT_TXT], 'a'),
'avail-easyconfig-templates':
(("Show all template names and template constants "
"that can be used in easyconfigs"), None, 'store_true', False),
'dep-graph': ("Create dependency graph", None, "store", None, {
'metavar': 'depgraph.<ext>'
}),
'list-easyblocks': ("Show list of available easyblocks", 'choice',
'store_or_None', 'simple',
['simple', 'detailed']),
'list-toolchains': ("Show list of known toolchains", None,
'store_true', False),
'search':
("Search for easyconfig files in the robot directory, print full paths",
None, 'store', None, {
'metavar': 'STR'
}),
'search-short':
("Search for easyconfig files in the robot directory, print short paths",
None, 'store', None, 'S', {
'metavar': 'STR'
}),
})
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def regtest_options(self):
# regression test options
descr = ("Regression test options",
"Run and control an EasyBuild regression test.")
opts = OrderedDict({
'aggregate-regtest':
("Collect all the xmls inside the given directory and generate a single file",
None, 'store', None, {
'metavar': 'DIR'
}),
'dump-test-report': ("Dump test report to specified path", None,
'store_or_None', 'test_report.md'),
'github-user': ("GitHub username", None, 'store', None),
'regtest': ("Enable regression test mode", None, 'store_true',
False),
'regtest-output-dir': ("Set output directory for test-run", None,
'store', None, {
'metavar': 'DIR'
}),
'sequential':
("Specify this option if you want to prevent parallel build", None,
'store_true', False),
'upload-test-report':
("Upload full test report as a gist on GitHub", None, 'store_true',
False),
'test-report-env-filter':
("Regex used to filter out variables in environment dump of test report",
None, 'regex', None),
})
self.log.debug("regtest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def easyconfig_options(self):
# easyconfig options (to be passed to easyconfig instance)
descr = ("Options for Easyconfigs",
"Options to be passed to all Easyconfig.")
opts = None
self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyconfig')
def easyblock_options(self):
# easyblock options (to be passed to easyblock instance)
descr = ("Options for Easyblocks",
"Options to be passed to all Easyblocks.")
opts = None
self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyblock')
def unittest_options(self):
# unittest options
descr = ("Unittest options",
"Options dedicated to unittesting (experts only).")
opts = OrderedDict({
'file': ("Log to this file in unittest mode", None, 'store', None),
})
self.log.debug("unittest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='unittest')
def validate(self):
"""Additional validation of options"""
error_cnt = 0
for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']:
val = getattr(self.options, opt.replace('-', '_'))
if val and len(val) != 2:
self.log.warning('--%s requires NAME,VERSION (given %s)' %
(opt, ','.join(val)))
error_cnt += 1
if self.options.umask:
umask_regex = re.compile('^[0-7]{3}$')
if not umask_regex.match(self.options.umask):
self.log.warning(
"--umask value should be 3 digits (0-7) (regex pattern '%s')"
% umask_regex.pattern)
error_cnt += 1
if error_cnt > 0:
self.log.error(
"Found %s problems validating the options, treating warnings above as fatal."
% error_cnt)
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_cfgfile_constants,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict(
) # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
self.log.error(
"Required support for using GitHub API is not available (see warnings)."
)
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
self.log.error(
"Python 'keyring' module required for obtaining GitHub token is not available."
)
if self.options.github_user is None:
self.log.error(
"No GitHub user name provided, required for fetching GitHub token."
)
token = fetch_github_token(self.options.github_user)
if token is None:
self.log.error(
"Failed to obtain required GitHub token for user '%s'" %
self.options.github_user)
self._postprocess_config()
def _postprocess_config(self):
"""Postprocessing of configuration options"""
if self.options.prefix is not None:
# prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account
# in the legacy-style configuration, repository is initialised in configuration file itself
for dest in [
'installpath', 'buildpath', 'sourcepath', 'repository',
'repositorypath'
]:
if not self.options._action_taken.get(dest, False):
if dest == 'repository':
setattr(self.options, dest, DEFAULT_REPOSITORY)
elif dest == 'repositorypath':
setattr(self.options, dest, [
mk_full_default_path(dest,
prefix=self.options.prefix)
])
else:
setattr(
self.options, dest,
mk_full_default_path(dest,
prefix=self.options.prefix))
# LEGACY this line is here for oldstyle config reasons
self.options._action_taken[dest] = True
if self.options.pretend:
self.options.installpath = get_pretend_installpath()
if self.options.robot is not None:
# paths specified to --robot have preference over --robot-paths
# keep both values in sync if robot is enabled, which implies enabling dependency resolver
self.options.robot_paths = self.options.robot + self.options.robot_paths
self.options.robot = self.options.robot_paths
def _postprocess_list_avail(self):
"""Create all the additional info that can be requested (exit at the end)"""
msg = ''
# dump supported configuration file constants
if self.options.avail_cfgfile_constants:
msg += self.avail_cfgfile_constants()
# dump possible easyconfig params
if self.options.avail_easyconfig_params:
msg += avail_easyconfig_params(
self.options.easyblock, self.options.avail_easyconfig_params)
# dump easyconfig template options
if self.options.avail_easyconfig_templates:
msg += template_documentation()
# dump easyconfig constant options
if self.options.avail_easyconfig_constants:
msg += constant_documentation()
# dump easyconfig license options
if self.options.avail_easyconfig_licenses:
msg += license_documentation()
# dump available easyblocks
if self.options.list_easyblocks:
msg += self.avail_easyblocks()
# dump known toolchains
if self.options.list_toolchains:
msg += self.avail_toolchains()
# dump known repository types
if self.options.avail_repositories:
msg += self.avail_repositories()
# dump supported modules tools
if self.options.avail_modules_tools:
msg += self.avail_list('modules tools', avail_modules_tools())
# dump supported module naming schemes
if self.options.avail_module_naming_schemes:
msg += self.avail_list('module naming schemes',
avail_module_naming_schemes())
# dump default moduleclasses with description
if self.options.show_default_moduleclasses:
msg += self.show_default_moduleclasses()
if self.options.unittest_file:
self.log.info(msg)
else:
print msg
sys.exit(0)
def avail_cfgfile_constants(self):
"""
Return overview of constants supported in configuration files.
"""
lines = [
"Constants available (only) in configuration files:",
"syntax: %(CONSTANT_NAME)s",
]
for section in self.go_cfg_constants:
lines.append('')
if section != self.DEFAULTSECT:
section_title = "only in '%s' section:" % section
lines.append(section_title)
for cst_name, (cst_value, cst_help) in sorted(
self.go_cfg_constants[section].items()):
lines.append("* %s: %s [value: %s]" %
(cst_name, cst_help, cst_value))
return '\n'.join(lines)
def avail_classes_tree(self, classes, classNames, detailed, depth=0):
"""Print list of classes as a tree."""
txt = []
for className in classNames:
classInfo = classes[className]
if detailed:
txt.append("%s|-- %s (%s)" %
("| " * depth, className, classInfo['module']))
else:
txt.append("%s|-- %s" % ("| " * depth, className))
if 'children' in classInfo:
txt.extend(
self.avail_classes_tree(classes, classInfo['children'],
detailed, depth + 1))
return txt
def avail_easyblocks(self):
"""Get a class tree for easyblocks."""
detailed = self.options.list_easyblocks == "detailed"
module_regexp = re.compile(r"^([^_].*)\.py$")
# finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
search_toolchain('')
for package in [
"easybuild.easyblocks", "easybuild.easyblocks.generic"
]:
__import__(package)
# determine paths for this package
paths = sys.modules[package].__path__
# import all modules in these paths
for path in paths:
if os.path.exists(path):
for f in os.listdir(path):
res = module_regexp.match(f)
if res:
__import__("%s.%s" % (package, res.group(1)))
def add_class(classes, cls):
"""Add a new class, and all of its subclasses."""
children = cls.__subclasses__()
classes.update({
cls.__name__: {
'module': cls.__module__,
'children': [x.__name__ for x in children]
}
})
for child in children:
add_class(classes, child)
roots = [EasyBlock, Extension]
classes = {}
for root in roots:
add_class(classes, root)
# Print the tree, start with the roots
txt = []
for root in roots:
root = root.__name__
if detailed:
txt.append("%s (%s)" % (root, classes[root]['module']))
else:
txt.append("%s" % root)
if 'children' in classes[root]:
txt.extend(
self.avail_classes_tree(classes, classes[root]['children'],
detailed))
txt.append("")
return "\n".join(txt)
def avail_toolchains(self):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
all_tcs_names = [x.NAME for x in all_tcs]
tclist = sorted(zip(all_tcs_names, all_tcs))
txt = ["List of known toolchains (toolchainname: module[,module...]):"]
for (tcname, tcc) in tclist:
tc = tcc(
version='1.2.3'
) # version doesn't matter here, but something needs to be there
tc_elems = nub(
sorted([e for es in tc.definition().values() for e in es]))
txt.append("\t%s: %s" % (tcname, ', '.join(tc_elems)))
return '\n'.join(txt)
def avail_repositories(self):
"""Show list of known repository types."""
repopath_defaults = mk_full_default_path('repositorypath')
all_repos = avail_repositories(check_useable=False)
usable_repos = avail_repositories(check_useable=True).keys()
indent = ' ' * 2
txt = ['All avaliable repository types']
repos = sorted(all_repos.keys())
for repo in repos:
if repo in usable_repos:
missing = ''
else:
missing = ' (*not usable*, something is missing (e.g. a required Python module))'
if repo in repopath_defaults:
default = ' (default arguments: %s)' % ', '.join(
repopath_defaults[repo])
else:
default = ' (no default arguments)'
txt.append("%s* %s%s%s" % (indent, repo, default, missing))
txt.append("%s%s" % (indent * 3, all_repos[repo].DESCRIPTION))
return "\n".join(txt)
def avail_list(self, name, items):
"""Show list of available values passed by argument."""
return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items))
def show_default_moduleclasses(self):
"""Show list of default moduleclasses and description."""
txt = ["Default available moduleclasses"]
indent = " " * 2
maxlen = max([len(x[0])
for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space
for name, descr in DEFAULT_MODULECLASSES:
txt.append("%s%s:%s%s" % (indent, name,
(" " * (maxlen - len(name))), descr))
return "\n".join(txt)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error(
"No robot paths specified, and unable to determine easybuild-easyconfigs install path."
)
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
if robot_path is None:
robot_path = []
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info(
"Extended list of robot paths with paths for installed easyconfigs: %s"
% robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest':
options.aggregate_regtest,
'allow_modules_tool_mismatch':
options.allow_modules_tool_mismatch,
'check_osdeps':
not options.ignore_osdeps,
'cleanup_builddir':
options.cleanup_builddir,
'command_line':
eb_command_line,
'debug':
options.debug,
'dry_run':
options.dry_run,
'easyblock':
options.easyblock,
'experimental':
options.experimental,
'force':
options.force,
'github_user':
options.github_user,
'group':
options.group,
'ignore_dirs':
options.ignore_dirs,
'modules_footer':
options.modules_footer,
'only_blocks':
options.only_blocks,
'recursive_mod_unload':
options.recursive_module_unload,
'regtest_output_dir':
options.regtest_output_dir,
'retain_all_deps':
retain_all_deps,
'robot_path':
robot_path,
'sequential':
options.sequential,
'silent':
testing,
'set_gid_bit':
options.set_gid_bit,
'skip':
options.skip,
'skip_test_cases':
options.skip_test_cases,
'sticky_bit':
options.sticky_bit,
'stop':
options.stop,
'umask':
options.umask,
'valid_module_classes':
module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate':
not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list()
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path,
query,
short=not options.search,
ignore_dirs=ignore_dirs,
silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
pr_files = fetch_easyconfigs_from_pr(
options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files
if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [
obtain_path(build_specs,
easyconfigs_paths,
try_to_generate=try_to_generate,
exit_on_error=not testing)
]
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(
orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug(
"Looking for missing easyconfig files (%d left) in %s..." %
(len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path,
topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" %
(orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [
d for d in dirnames if not d in options.ignore_dirs
]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" %
(path, err))
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
"That's not wise, so let's end this here.\n" \
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key('name'):
paths = [obtain_path(software_build_specs, easyconfigs_paths,
try_to_generate=try_to_generate, exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path,
easyconfigs_map[orig_path]))
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
class EasyBuildOptions(GeneralOption):
"""Easybuild generaloption class"""
VERSION = this_is_easybuild()
DEFAULT_LOGLEVEL = 'INFO'
DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:]
if os.path.exists(DEFAULT_USER_CFGFILE):
DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE)
ALLOPTSMANDATORY = False # allow more than one argument
CONFIGFILES_RAISE_MISSING = True # don't allow non-existing config files to be specified
def __init__(self, *args, **kwargs):
"""Constructor."""
self.default_repositorypath = [mk_full_default_path('repositorypath')]
self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR,
robot_path=None) or []
# set up constants to seed into config files parser, by section
self.go_cfg_constants = {
self.DEFAULTSECT: {
'DEFAULT_REPOSITORYPATH':
(self.default_repositorypath[0],
"Default easyconfigs repository path"),
'DEFAULT_ROBOT_PATHS':
(os.pathsep.join(self.default_robot_paths),
"List of default robot paths ('%s'-separated)" % os.pathsep),
}
}
# update or define go_configfiles_initenv in named arguments to pass to parent constructor
go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {})
for section, constants in self.go_cfg_constants.items():
constants = dict([(name, value)
for (name, (value, _)) in constants.items()])
go_cfg_initenv.setdefault(section, {}).update(constants)
super(EasyBuildOptions, self).__init__(*args, **kwargs)
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [run.IGNORE, run.WARN, run.ERROR]
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
'dry-run': ("Print build overview incl. dependencies (full paths)",
None, 'store_true', False),
'dry-run-short':
("Print build overview incl. dependencies (short paths)", None,
'store_true', False, 'D'),
'force':
("Force to rebuild software even if it's already installed (i.e. if it can be found as module)",
None, 'store_true', False, 'f'),
'job': ("Submit the build as a job", None, 'store_true', False),
'logtostdout': ("Redirect main log to stdout", None, 'store_true',
False, 'l'),
'only-blocks': ("Only build listed blocks", None, 'extend', None,
'b', {
'metavar': 'BLOCKS'
}),
'robot':
("Enable dependency resolution, using easyconfigs in specified paths",
'pathlist', 'store_or_None', [], 'r', {
'metavar': 'PATH[%sPATH]' % os.pathsep
}),
'robot-paths':
("Additional paths to consider by robot for easyconfigs (--robot paths get priority)",
'pathlist', 'add_flex', self.default_robot_paths, {
'metavar': 'PATH[%sPATH]' % os.pathsep
}),
'skip':
("Skip existing software (useful for installing additional packages)",
None, 'store_true', False, 'k'),
'stop': ("Stop the installation after certain step", 'choice',
'store_or_None', SOURCE_STEP, 's', all_stops),
'strict': ("Set strictness level", 'choice', 'store',
DEFAULT_STRICT, strictness_options),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def software_options(self):
# software build options
descr = ("Software search and build options", (
"Specify software search and build options: EasyBuild will search for a "
"matching easyconfig and build it. When called with the try prefix "
"(i.e. --try-X ), EasyBuild will search for a matching easyconfig "
"and if none are found, try to generate one based on a close matching one "
"(NOTE: --try-X is best effort, it might produce wrong builds!)"))
opts = OrderedDict({
'amend':
(("Specify additional search and build parameters (can be used multiple times); "
"for example: versionprefix=foo or patches=one.patch,two.patch)"
), None, 'append', None, {
'metavar': 'VAR=VALUE[,VALUE]'
}),
'software':
("Search and build software with given name and version", None,
'extend', None, {
'metavar': 'NAME,VERSION'
}),
'software-name': ("Search and build software with given name",
None, 'store', None, {
'metavar': 'NAME'
}),
'software-version':
("Search and build software with given version", None, 'store',
None, {
'metavar': 'VERSION'
}),
'toolchain':
("Search and build with given toolchain (name and version)", None,
'extend', None, {
'metavar': 'NAME,VERSION'
}),
'toolchain-name': ("Search and build with given toolchain name",
None, 'store', None, {
'metavar': 'NAME'
}),
'toolchain-version':
("Search and build with given toolchain version", None, 'store',
None, {
'metavar': 'VERSION'
}),
})
longopts = opts.keys()
for longopt in longopts:
hlp = opts[longopt][0]
hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:])
opts["try-%s" % longopt] = (hlp, ) + opts[longopt][1:]
# additional options that don't need a --try equivalent
opts.update({
'from-pr':
("Obtain easyconfigs from specified PR", int, 'store', None, {
'metavar': 'PR#'
}),
})
self.log.debug("software_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
opts = OrderedDict({
'allow-modules-tool-mismatch':
("Allow mismatch of modules tool and definition of 'module' function",
None, 'store_true', False),
'cleanup-builddir':
("Cleanup build dir after successful installation.", None,
'store_true', True),
'deprecated':
("Run pretending to be (future) version, to test removal of deprecated code.",
None, 'store', None),
'download-timeout':
("Timeout for initiating downloads (in seconds)", float, 'store',
None),
'easyblock':
("easyblock to use for processing the spec file or dumping the options",
None, 'store', None, 'e', {
'metavar': 'CLASS'
}),
'experimental':
("Allow experimental code (with behaviour that can be changed/removed at any given time).",
None, 'store_true', False),
'group':
("Group to be used for software installations (only verified, not set)",
None, 'store', None),
'hidden':
("Install 'hidden' module file(s) by prefixing their name with '.'",
None, 'store_true', False),
'ignore-osdeps': ("Ignore any listed OS dependencies", None,
'store_true', False),
'filter-deps':
("Comma separated list of dependencies that you DON'T want to install with EasyBuild, "
"because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)",
'strlist', 'extend', None),
'hide-deps':
("Comma separated list of dependencies that you want automatically hidden, "
"(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None),
'module-only':
("Only generate module file(s); skip all steps except for %s" %
', '.join(MODULE_ONLY_STEPS), None, 'store_true', False),
'optarch':
("Set architecture optimization, overriding native architecture optimizations",
None, 'store', None),
'pretend':
(("Does the build/installation in a test directory located in $HOME/easybuildinstall"
), None, 'store_true', False, 'p'),
'set-gid-bit': ("Set group ID bit on newly created directories",
None, 'store_true', False),
'sticky-bit': ("Set sticky bit on newly created directories", None,
'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true',
False, 't'),
'umask':
("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
None, 'store', None),
'update-modules-tool-cache':
("Update modules tool cache file(s) after generating module file",
None, 'store_true', False),
})
self.log.debug("override_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def config_options(self):
# config options
descr = ("Configuration options", "Configure EasyBuild behavior.")
opts = OrderedDict({
'avail-module-naming-schemes': (
"Show all supported module naming schemes",
None,
'store_true',
False,
),
'avail-modules-tools': (
"Show all supported module tools",
None,
"store_true",
False,
),
'avail-repositories': (
"Show all repository types (incl. non-usable)",
None,
"store_true",
False,
),
'buildpath': ("Temporary build path", None, 'store',
mk_full_default_path('buildpath')),
'external-modules-metadata':
("List of files specifying metadata for external modules (INI format)",
'strlist', 'store', []),
'ignore-dirs':
("Directory names to ignore when searching for files/dirs",
'strlist', 'store', ['.git', '.svn']),
'installpath': ("Install path for software and modules", None,
'store', mk_full_default_path('installpath')),
'installpath-modules':
("Install path for modules (if None, combine --installpath and --subdir-modules)",
None, 'store', None),
'installpath-software':
("Install path for software (if None, combine --installpath and --subdir-software)",
None, 'store', None),
# purposely take a copy for the default logfile format
'logfile-format':
("Directory name and format of the log file", 'strtuple', 'store',
DEFAULT_LOGFILE_FORMAT[:], {
'metavar': 'DIR,FORMAT'
}),
'module-naming-scheme':
("Module naming scheme", 'choice', 'store', DEFAULT_MNS,
sorted(avail_module_naming_schemes().keys())),
'module-syntax': ("Syntax to be used for module files", 'choice',
'store', DEFAULT_MODULE_SYNTAX,
sorted(avail_module_generators().keys())),
'moduleclasses':
(("Extend supported module classes "
"(For more info on the default classes, use --show-default-moduleclasses)"
), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]),
'modules-footer':
("Path to file containing footer to be added to all generated module files",
None, 'store_or_None', None, {
'metavar': "PATH"
}),
'modules-tool': ("Modules tool to use", 'choice', 'store',
DEFAULT_MODULES_TOOL,
sorted(avail_modules_tools().keys())),
'prefix':
(("Change prefix for buildpath, installpath, sourcepath and repositorypath "
"(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store',
None),
'recursive-module-unload':
("Enable generating of modules that unload recursively.", None,
'store_true', False),
'repository': ("Repository type, using repositorypath", 'choice',
'store', DEFAULT_REPOSITORY,
sorted(avail_repositories().keys())),
'repositorypath':
(("Repository path, used by repository "
"(is passed as list of arguments to create the repository instance). "
"For more info, use --avail-repositories."), 'strlist', 'store',
self.default_repositorypath),
'sourcepath':
("Path(s) to where sources should be downloaded (string, colon-separated)",
None, 'store', mk_full_default_path('sourcepath')),
'subdir-modules': ("Installpath subdir for modules", None, 'store',
DEFAULT_PATH_SUBDIRS['subdir_modules']),
'subdir-software': ("Installpath subdir for software", None,
'store',
DEFAULT_PATH_SUBDIRS['subdir_software']),
'suffix-modules-path': ("Suffix for module files install path",
None, 'store', GENERAL_CLASS),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
'testoutput':
("Path to where a job should place the output (to be set within jobscript)",
None, 'store', None),
'tmp-logdir':
("Log directory where temporary log files are stored", None,
'store', None),
'tmpdir': ('Directory to use for temporary storage', None, 'store',
None),
})
self.log.debug("config_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def informative_options(self):
# informative options
descr = ("Informative options", "Obtain information about EasyBuild.")
opts = OrderedDict({
'avail-cfgfile-constants':
("Show all constants that can be used in configuration files",
None, 'store_true', False),
'avail-easyconfig-constants':
("Show all constants that can be used in easyconfigs", None,
'store_true', False),
'avail-easyconfig-licenses':
("Show all license constants that can be used in easyconfigs",
None, 'store_true', False),
'avail-easyconfig-params':
(("Show all easyconfig parameters (include "
"easyblock-specific ones by using -e)"), 'choice',
'store_or_None', FORMAT_TXT, [FORMAT_RST, FORMAT_TXT], 'a'),
'avail-easyconfig-templates':
(("Show all template names and template constants "
"that can be used in easyconfigs"), None, 'store_true', False),
'dep-graph': ("Create dependency graph", None, "store", None, {
'metavar': 'depgraph.<ext>'
}),
'list-easyblocks': ("Show list of available easyblocks", 'choice',
'store_or_None', 'simple',
['simple', 'detailed']),
'list-toolchains': ("Show list of known toolchains", None,
'store_true', False),
'search':
("Search for easyconfig files in the robot directory, print full paths",
None, 'store', None, {
'metavar': 'STR'
}),
'search-short':
("Search for easyconfig files in the robot directory, print short paths",
None, 'store', None, 'S', {
'metavar': 'STR'
}),
'show-default-configfiles': ("Show list of default config files",
None, 'store_true', False),
'show-default-moduleclasses':
("Show default module classes with description", None,
'store_true', False),
})
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def regtest_options(self):
# regression test options
descr = ("Regression test options",
"Run and control an EasyBuild regression test.")
opts = OrderedDict({
'aggregate-regtest':
("Collect all the xmls inside the given directory and generate a single file",
None, 'store', None, {
'metavar': 'DIR'
}),
'dump-test-report': ("Dump test report to specified path", None,
'store_or_None', 'test_report.md'),
'github-user': ("GitHub username", None, 'store', None),
'regtest': ("Enable regression test mode", None, 'store_true',
False),
'regtest-output-dir': ("Set output directory for test-run", None,
'store', None, {
'metavar': 'DIR'
}),
'sequential':
("Specify this option if you want to prevent parallel build", None,
'store_true', False),
'upload-test-report':
("Upload full test report as a gist on GitHub", None, 'store_true',
False),
'test-report-env-filter':
("Regex used to filter out variables in environment dump of test report",
None, 'regex', None),
})
self.log.debug("regtest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def easyconfig_options(self):
# easyconfig options (to be passed to easyconfig instance)
descr = ("Options for Easyconfigs",
"Options to be passed to all Easyconfig.")
opts = None
self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyconfig')
def easyblock_options(self):
# easyblock options (to be passed to easyblock instance)
descr = ("Options for Easyblocks",
"Options to be passed to all Easyblocks.")
opts = None
self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='easyblock')
def unittest_options(self):
# unittest options
descr = ("Unittest options",
"Options dedicated to unittesting (experts only).")
opts = OrderedDict({
'file': ("Log to this file in unittest mode", None, 'store', None),
})
self.log.debug("unittest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='unittest')
def validate(self):
"""Additional validation of options"""
error_msgs = []
for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']:
val = getattr(self.options, opt.replace('-', '_'))
if val and len(val) != 2:
msg = "--%s requires NAME,VERSION (given %s)" % (opt,
','.join(val))
error_msgs.append(msg)
if self.options.umask:
umask_regex = re.compile('^[0-7]{3}$')
if not umask_regex.match(self.options.umask):
msg = "--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern
error_msgs.append(msg)
# subdir options must be relative
for typ in ['modules', 'software']:
subdir_opt = 'subdir_%s' % typ
val = getattr(self.options, subdir_opt)
if os.path.isabs(getattr(self.options, subdir_opt)):
msg = "Configuration option '%s' must specify a *relative* path (use 'installpath-%s' instead?): '%s'"
msg = msg % (subdir_opt, typ, val)
error_msgs.append(msg)
if error_msgs:
raise EasyBuildError("Found problems validating the options: %s",
'\n'.join(error_msgs))
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_cfgfile_constants,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
self.options.show_default_configfiles,
]):
build_easyconfig_constants_dict(
) # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
raise EasyBuildError(
"Required support for using GitHub API is not available (see warnings)."
)
if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
raise EasyBuildError(
"Generating Lua module files requires Lmod as modules tool.")
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
raise EasyBuildError(
"Python 'keyring' module required for obtaining GitHub token is not available."
)
if self.options.github_user is None:
raise EasyBuildError(
"No GitHub user name provided, required for fetching GitHub token."
)
token = fetch_github_token(self.options.github_user)
if token is None:
raise EasyBuildError(
"Failed to obtain required GitHub token for user '%s'",
self.options.github_user)
self._postprocess_external_modules_metadata()
self._postprocess_config()
def _postprocess_external_modules_metadata(self):
"""Parse file(s) specifying metadata for external modules."""
# leave external_modules_metadata untouched if no files are provided
if not self.options.external_modules_metadata:
self.log.debug("No metadata provided for external modules.")
return
parsed_external_modules_metadata = ConfigObj()
for path in self.options.external_modules_metadata:
if os.path.exists(path):
self.log.debug("Parsing %s with external modules metadata",
path)
try:
parsed_external_modules_metadata.merge(ConfigObj(path))
except ConfigObjError, err:
raise EasyBuildError(
"Failed to parse %s with external modules metadata: %s",
path, err)
else:
raise EasyBuildError(
"Specified path for file with external modules metadata does not exist: %s",
path)
# make sure name/version values are always lists, make sure they're equal length
for mod, entry in parsed_external_modules_metadata.items():
for key in ['name', 'version']:
if isinstance(entry.get(key), basestring):
entry[key] = [entry[key]]
self.log.debug(
"Transformed external module metadata value %s for %s into a single-value list: %s",
key, mod, entry[key])
# if both names and versions are available, lists must be of same length
names, versions = entry.get('name'), entry.get('version')
if names is not None and versions is not None and len(
names) != len(versions):
raise EasyBuildError(
"Different length for lists of names/versions in metadata for external module %s: "
"names: %s; versions: %s", mod, names, versions)
self.options.external_modules_metadata = parsed_external_modules_metadata
self.log.debug("External modules metadata: %s",
self.options.external_modules_metadata)