def regtest_options(self):
# regression test options
descr = ("Regression test options",
"Run and control an EasyBuild regression test.")
opts = OrderedDict({
'aggregate-regtest':
("Collect all the xmls inside the given directory and generate a single file",
None, 'store', None, {
'metavar': 'DIR'
}),
'dump-test-report': ("Dump test report to specified path", None,
'store_or_None', 'test_report.md'),
'github-user': ("GitHub username", None, 'store', None),
'regtest': ("Enable regression test mode", None, 'store_true',
False),
'regtest-output-dir': ("Set output directory for test-run", None,
'store', None, {
'metavar': 'DIR'
}),
'sequential':
("Specify this option if you want to prevent parallel build", None,
'store_true', False),
'upload-test-report':
("Upload full test report as a gist on GitHub", None, 'store_true',
False),
'test-report-env-filter':
("Regex used to filter out variables in environment dump of test report",
None, 'regex', None),
})
self.log.debug("regtest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def informative_options(self):
# informative options
descr = ("Informative options", "Obtain information about EasyBuild.")
opts = OrderedDict({
'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs",
None, 'store_true', False),
'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs",
None, 'store_true', False),
'avail-easyconfig-params': (("Show all easyconfig parameters (include "
"easyblock-specific ones by using -e)"),
None, "store_true", False, 'a'),
'avail-easyconfig-templates': (("Show all template names and template constants "
"that can be used in easyconfigs"),
None, 'store_true', False),
'dep-graph': ("Create dependency graph",
None, "store", None, {'metavar': 'depgraph.<ext>'}),
'list-easyblocks': ("Show list of available easyblocks",
'choice', 'store_or_None', 'simple', ['simple', 'detailed']),
'list-toolchains': ("Show list of known toolchains",
None, 'store_true', False),
'search': ("Search for easyconfig files in the robot directory, print full paths",
None, 'store', None, {'metavar': 'STR'}),
'search-short': ("Search for easyconfig files in the robot directory, print short paths",
None, 'store', None, 'S', {'metavar': 'STR'}),
})
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [run.IGNORE, run.WARN, run.ERROR]
try:
default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0]
except:
self.log.warning("basic_options: unable to determine default easyconfig path")
default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False),
'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'),
'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)",
None, 'store_true', False, 'f'),
'job': ("Submit the build as a job", None, 'store_true', False),
'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'),
'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', {'metavar': 'BLOCKS'}),
'robot': ("Path(s) to search for easyconfigs for missing dependencies (colon-separated)" ,
None, 'store_or_None', default_robot_path, 'r', {'metavar': 'PATH'}),
'skip': ("Skip existing software (useful for installing additional packages)",
None, 'store_true', False, 'k'),
'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops),
'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def software_options(self):
# software build options
descr = ("Software search and build options",
("Specify software search and build options: EasyBuild will search for a "
"matching easyconfig and build it. When called with the try prefix "
"(i.e. --try-X ), EasyBuild will search for a matching easyconfig "
"and if none are found, try to generate one based on a close matching one "
"(NOTE: --try-X is best effort, it might produce wrong builds!)")
)
opts = OrderedDict({
'software-name':("Search and build software with name",
None, 'store', None, {'metavar':'NAME'}),
'software-version':("Search and build software with version",
None, 'store', None, {'metavar':'VERSION'}),
'toolchain':("Search and build with toolchain (name and version)",
None, 'extend', None, {'metavar':'NAME,VERSION'}),
'toolchain-name':("Search and build with toolchain name",
None, 'store', None, {'metavar':'NAME'}),
'toolchain-version':("Search and build with toolchain version",
None, 'store', None, {'metavar':'VERSION'}),
'amend':(("Specify additional search and build parameters (can be used multiple times); "
"for example: versionprefix=foo or patches=one.patch,two.patch)"),
None, 'append', None, {'metavar':'VAR=VALUE[,VALUE]'}),
})
longopts = opts.keys()
for longopt in longopts:
hlp = opts[longopt][0]
hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:])
opts["try-%s" % longopt] = (hlp,) + opts[longopt][1:]
self.log.debug("software_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [filetools.IGNORE, filetools.WARN, filetools.ERROR]
try:
default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0]
except:
self.log.warning("basic_options: unable to determine default easyconfig path")
default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
"only-blocks":("Only build listed blocks",
None, "extend", None, "b", {'metavar':"BLOCKS"}),
"force":(("Force to rebuild software even if it's already installed "
"(i.e. if it can be found as module)"),
None, "store_true", False, "f"),
"job":("Submit the build as a job", None, "store_true", False),
"skip":("Skip existing software (useful for installing additional packages)",
None, "store_true", False, "k"),
"robot":("Path to search for easyconfigs for missing dependencies." ,
None, "store_or_None", default_robot_path, "r", {'metavar':"PATH"}),
"stop":("Stop the installation after certain step",
"choice", "store_or_None", "source", "s", all_stops),
"strict":("Set strictness level",
"choice", "store", filetools.WARN, strictness_options),
"logtostdout":("Redirect main log to stdout", None, "store_true", False, "l"),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
opts = OrderedDict({
'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function",
None, 'store_true', False),
'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True),
'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.",
None, 'store', None),
'easyblock': ("easyblock to use for processing the spec file or dumping the options",
None, 'store', None, 'e', {'metavar': 'CLASS'}),
'experimental': ("Allow experimental code (with behaviour that can be changed or removed at any given time).",
None, 'store_true', False),
'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),
'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
'oldstyleconfig': ("Look for and use the oldstyle configuration file.",
None, 'store_true', True),
'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
None, 'store_true', False, 'p'),
'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False),
'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
None, 'store', None),
})
self.log.debug("override_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def unittest_options(self):
# unittest options
descr = ("Unittest options", "Options dedicated to unittesting (experts only).")
opts = OrderedDict({
'file': ("Log to this file in unittest mode", None, 'store', None),
})
self.log.debug("unittest_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr, prefix='unittest')
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = [
('framework', [('easybuild/framework', True),
('easybuild/tools', True)]),
('easyblocks', [('easybuild/easyblocks', True)]),
('easyconfigs', [('easybuild/easyconfigs', False)]),
]
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(
os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg:
sel_dirs = [
x for x in installed_dirs
if x.startswith('easybuild_%s' % pkg)
]
if not len(sel_dirs) == 1:
self.log.error(
"Failed to isolate installed egg dir for easybuild-%s"
% pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append(
(os.path.join(sel_dirs[0], subdir), True))
except OSError, err:
self.log.error(
"Failed to determine sanity check dir paths: %s" % err)
def print_pr_summary(pr_data):
"""Print PR summary."""
keys = OrderedDict([
('html_url', 'URL'),
('title', None),
(('user', 'login'), "issued by"),
(('head', 'ref'), "branch"),
(('base', 'ref'), "target branch"),
('state', "status"),
(('milestone', 'title'), None),
])
target = '%s/%s' % (pr_data['base']['repo']['owner']['login'],
pr_data['base']['repo']['name'])
print "\nSummary for %s PR #%s:\n" % (target, pr_data['number'])
for key in keys:
print_pr_info(pr_data, key, label=keys[key])
print ''
def basic_options(self):
"""basic runtime options"""
all_stops = [x[0] for x in EasyBlock.get_steps()]
strictness_options = [run.IGNORE, run.WARN, run.ERROR]
descr = ("Basic options", "Basic runtime options for EasyBuild.")
opts = OrderedDict({
'dry-run': ("Print build overview incl. dependencies (full paths)",
None, 'store_true', False),
'dry-run-short':
("Print build overview incl. dependencies (short paths)", None,
'store_true', False, 'D'),
'force':
("Force to rebuild software even if it's already installed (i.e. if it can be found as module)",
None, 'store_true', False, 'f'),
'job': ("Submit the build as a job", None, 'store_true', False),
'logtostdout': ("Redirect main log to stdout", None, 'store_true',
False, 'l'),
'only-blocks': ("Only build listed blocks", None, 'extend', None,
'b', {
'metavar': 'BLOCKS'
}),
'robot':
("Enable dependency resolution, using easyconfigs in specified paths",
'pathlist', 'store_or_None', [], 'r', {
'metavar': 'PATH[%sPATH]' % os.pathsep
}),
'robot-paths':
("Additional paths to consider by robot for easyconfigs (--robot paths get priority)",
'pathlist', 'store', self.default_robot_paths, {
'metavar': 'PATH[%sPATH]' % os.pathsep
}),
'skip':
("Skip existing software (useful for installing additional packages)",
None, 'store_true', False, 'k'),
'stop': ("Stop the installation after certain step", 'choice',
'store_or_None', 'source', 's', all_stops),
'strict': ("Set strictness level", 'choice', 'store', run.WARN,
strictness_options),
})
self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def avail_easyconfig_params(easyblock, output_format):
"""
Compose overview of available easyconfig parameters, in specified format.
"""
params = copy.deepcopy(DEFAULT_CONFIG)
# include list of extra parameters (if any)
extra_params = {}
app = get_easyblock_class(easyblock, default_fallback=False)
if app is not None:
extra_params = app.extra_options()
params.update(extra_params)
# compose title
title = "Available easyconfig parameters"
if extra_params:
title += " (* indicates specific to the %s easyblock)" % app.__name__
# group parameters by category
grouped_params = OrderedDict()
for category in sorted_categories():
# exclude hidden parameters
if category[1].upper() in [HIDDEN]:
continue
grpname = category[1]
grouped_params[grpname] = {}
for name, (dflt, descr, cat) in params.items():
if cat == category:
if name in extra_params:
# mark easyblock-specific parameters
name = '%s*' % name
grouped_params[grpname].update({name: (descr, dflt)})
if not grouped_params[grpname]:
del grouped_params[grpname]
# compose output, according to specified format (txt, rst, ...)
avail_easyconfig_params_functions = {
FORMAT_RST: avail_easyconfig_params_rst,
FORMAT_TXT: avail_easyconfig_params_txt,
}
return avail_easyconfig_params_functions[output_format](title,
grouped_params)
def get_build_stats(app, starttime):
"""
Return build statistics for this build
"""
buildtime = round(time.time() - starttime, 2)
buildstats = OrderedDict([
('easybuild-framework_version', str(FRAMEWORK_VERSION)),
('easybuild-easyblocks_version', str(EASYBLOCKS_VERSION)),
('host', os.uname()[1]),
('platform' , platform.platform()),
('cpu_model', systemtools.get_cpu_model()),
('core_count', systemtools.get_core_count()),
('timestamp', int(time.time())),
('build_time', buildtime),
('install_size', app.det_installsize()),
])
return buildstats
def convert_to_help(opts, has_default=False):
"""
Converts the given list to a mapping of category -> [(name, help)] (OrderedDict)
@param: has_default, if False, add the DEFAULT_CONFIG list
"""
mapping = OrderedDict()
if isinstance(opts, dict):
opts = opts.items()
if not has_default:
defs = [(k, [def_val, descr, ALL_CATEGORIES[cat]]) for k, (def_val, descr, cat) in DEFAULT_CONFIG.items()]
opts = defs + opts
# sort opts
opts.sort()
for cat in sorted_categories():
mapping[cat[1]] = [(opt[0], "%s (default: %s)" % (opt[1][1], opt[1][0]))
for opt in opts if opt[1][2] == cat]
return mapping
def get_build_stats(app, start_time, command_line):
"""
Return build statistics for this build
"""
time_now = time.time()
build_time = round(time_now - start_time, 2)
buildstats = OrderedDict([
('easybuild-framework_version', str(FRAMEWORK_VERSION)),
('easybuild-easyblocks_version', str(EASYBLOCKS_VERSION)),
('timestamp', int(time_now)),
('build_time', build_time),
('install_size', app.det_installsize()),
('command_line', command_line),
('modules_tool', app.modules_tool.buildstats()),
])
for key, val in sorted(get_system_info().items()):
buildstats.update({key: val})
return buildstats
'',
'Please type the options to tune the gnu compilers, or ignore :':
os.getenv('F90FLAGS'),
}
f90_seq = os.getenv('F90_SEQ')
if not f90_seq:
# F90_SEQ is only defined when usempi is enabled
f90_seq = os.getenv('F90')
stdqa = OrderedDict([
(r'Confirm library .* is .*',
'y'), # this one needs to be tried first!
(r'.*fortran 90 compiler name .*\s*:\n\(suggestions\s*: .*\)',
f90_seq),
(r'.*fortran 90 compiler interfaced with .*\s*:\n\(suggestions\s*: .*\)',
f90_seq),
(r'Please type the ABSOLUTE name of .*library.*, or ignore\s*[:]*\s*[\n]*.*',
''),
(r'Please .* to save this draft configuration file :\n.*',
'%s.x' % self.conf_file),
])
no_qa = [
".*ignored.",
]
env.setvar('GMKTMP', self.builddir)
env.setvar('GMKFILE', self.conf_file)
run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa)
def config_options(self):
# config options
descr = ("Configuration options", "Configure EasyBuild behavior.")
opts = OrderedDict({
'avail-module-naming-schemes': (
"Show all supported module naming schemes",
None,
'store_true',
False,
),
'avail-modules-tools': (
"Show all supported module tools",
None,
"store_true",
False,
),
'avail-repositories': (
"Show all repository types (incl. non-usable)",
None,
"store_true",
False,
),
'buildpath': ("Temporary build path", None, 'store',
mk_full_default_path('buildpath')),
'ignore-dirs':
("Directory names to ignore when searching for files/dirs",
'strlist', 'store', ['.git', '.svn']),
'installpath': ("Install path for software and modules", None,
'store', mk_full_default_path('installpath')),
# purposely take a copy for the default logfile format
'logfile-format':
("Directory name and format of the log file", 'strtuple', 'store',
DEFAULT_LOGFILE_FORMAT[:], {
'metavar': 'DIR,FORMAT'
}),
'module-naming-scheme':
("Module naming scheme", 'choice', 'store', DEFAULT_MNS,
sorted(avail_module_naming_schemes().keys())),
'moduleclasses':
(("Extend supported module classes "
"(For more info on the default classes, use --show-default-moduleclasses)"
), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]),
'modules-footer':
("Path to file containing footer to be added to all generated module files",
None, 'store_or_None', None, {
'metavar': "PATH"
}),
'modules-tool': ("Modules tool to use", 'choice', 'store',
DEFAULT_MODULES_TOOL,
sorted(avail_modules_tools().keys())),
'prefix':
(("Change prefix for buildpath, installpath, sourcepath and repositorypath "
"(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store',
None),
'recursive-module-unload':
("Enable generating of modules that unload recursively.", None,
'store_true', False),
'repository': ("Repository type, using repositorypath", 'choice',
'store', DEFAULT_REPOSITORY,
sorted(avail_repositories().keys())),
'repositorypath':
(("Repository path, used by repository "
"(is passed as list of arguments to create the repository instance). "
"For more info, use --avail-repositories."), 'strlist', 'store',
[mk_full_default_path('repositorypath')]),
'show-default-moduleclasses':
("Show default module classes with description", None,
'store_true', False),
'sourcepath':
("Path(s) to where sources should be downloaded (string, colon-separated)",
None, 'store', mk_full_default_path('sourcepath')),
'subdir-modules': ("Installpath subdir for modules", None, 'store',
DEFAULT_PATH_SUBDIRS['subdir_modules']),
'subdir-software': ("Installpath subdir for software", None,
'store',
DEFAULT_PATH_SUBDIRS['subdir_software']),
'suffix-modules-path': ("Suffix for module files install path",
None, 'store', GENERAL_CLASS),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
'testoutput':
("Path to where a job should place the output (to be set within jobscript)",
None, 'store', None),
'tmp-logdir':
("Log directory where temporary log files are stored", None,
'store', None),
'tmpdir': ('Directory to use for temporary storage', None, 'store',
None),
})
self.log.debug("config_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# check whether easy-install.pth contains correct entries
easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth')
if os.path.exists(easy_install_pth):
easy_install_pth_txt = read_file(easy_install_pth)
for pkg in [p for p in self.easybuild_pkgs if p not in ['setuptools', 'vsc-install']]:
if pkg == 'vsc-base':
# don't include strict version check for vsc-base
pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M)
else:
major_minor_version = '.'.join(self.version.split('.')[:2])
pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M)
if not pkg_regex.search(easy_install_pth_txt):
raise EasyBuildError("Failed to find pattern '%s' in %s: %s",
pkg_regex.pattern, easy_install_pth, easy_install_pth_txt)
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = {
'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)],
'easybuild-easyblocks': [('easybuild/easyblocks', True)],
'easybuild-easyconfigs': [('easybuild/easyconfigs', False)],
}
if LooseVersion(self.version) >= LooseVersion('2.0'):
subdirs_by_pkg.update({
'vsc-base': [('vsc/utils', True)],
})
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs,
# since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg.items():
sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))]
if not len(sel_dirs) == 1:
raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True))
except OSError as err:
raise EasyBuildError("Failed to determine sanity check dir paths: %s", err)
# set of sanity check paths to check for EasyBuild
custom_paths = {
'files': ['bin/eb'],
'dirs': [self.pylibdir] + [[x, os.path.join(self.pylibdir, x)][y] for (x, y) in eb_dirs[setup_tool]],
}
# make sure we don't trip over deprecated behavior in old EasyBuild versions
eb_cmd = 'eb'
if LooseVersion(self.version) <= LooseVersion('1.16.0'):
eb_cmd = 'EASYBUILD_DEPRECATED=1.0 eb'
# set of sanity check commands to run for EasyBuild
custom_commands = [
# this may spit out a wrong version, but that should be safe to ignore
# occurs when the EasyBuild being used is newer than the EasyBuild being installed
(eb_cmd, '--version'),
(eb_cmd, '-a'),
(eb_cmd, '-e ConfigureMake -a'),
]
# (temporary) cleanse copy of initial environment to avoid conflict with (potentially) loaded EasyBuild module
self.real_initial_environ = copy.deepcopy(self.initial_environ)
for env_var in ['_LMFILES_', 'LOADEDMODULES']:
if env_var in self.initial_environ:
self.initial_environ.pop(env_var)
os.environ.pop(env_var)
self.log.debug("Unset $%s in current env and copy of original env to make sanity check work" % env_var)
super(EB_EasyBuildMeta, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# check whether easy-install.pth contains correct entries
easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth')
if os.path.exists(easy_install_pth):
easy_install_pth_txt = read_file(easy_install_pth)
for pkg in self.easybuild_pkgs:
if pkg == 'vsc-base':
# don't include strict version check for vsc-base
pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M)
else:
major_minor_version = '.'.join(self.version.split('.')[:2])
pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M)
if not pkg_regex.search(easy_install_pth_txt):
raise EasyBuildError("Failed to find pattern '%s' in %s: %s",
pkg_regex.pattern, easy_install_pth, easy_install_pth_txt)
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = {
'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)],
'easybuild-easyblocks': [('easybuild/easyblocks', True)],
'easybuild-easyconfigs': [('easybuild/easyconfigs', False)],
}
if LooseVersion(self.version) >= LooseVersion('2.0'):
subdirs_by_pkg.update({
'vsc-base': [('vsc/utils', True)],
})
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs,
# since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg.items():
sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))]
if not len(sel_dirs) == 1:
raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True))
except OSError, err:
raise EasyBuildError("Failed to determine sanity check dir paths: %s", err)
def override_options(self):
# override options
descr = ("Override options", "Override default EasyBuild behavior.")
opts = OrderedDict({
'allow-modules-tool-mismatch':
("Allow mismatch of modules tool and definition of 'module' function",
None, 'store_true', False),
'cleanup-builddir':
("Cleanup build dir after successful installation.", None,
'store_true', True),
'deprecated':
("Run pretending to be (future) version, to test removal of deprecated code.",
None, 'store', None),
'download-timeout':
("Timeout for initiating downloads (in seconds)", None, 'store',
None),
'easyblock':
("easyblock to use for processing the spec file or dumping the options",
None, 'store', None, 'e', {
'metavar': 'CLASS'
}),
'experimental':
("Allow experimental code (with behaviour that can be changed/removed at any given time).",
None, 'store_true', False),
'group':
("Group to be used for software installations (only verified, not set)",
None, 'store', None),
'hidden':
("Install 'hidden' module file(s) by prefixing their name with '.'",
None, 'store_true', False),
'ignore-osdeps': ("Ignore any listed OS dependencies", None,
'store_true', False),
'filter-deps':
("Comma separated list of dependencies that you DON'T want to install with EasyBuild, "
"because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)",
str, 'extend', None),
'oldstyleconfig':
("Look for and use the oldstyle configuration file.", None,
'store_true', True),
'optarch':
("Set architecture optimization, overriding native architecture optimizations",
None, 'store', None),
'pretend':
(("Does the build/installation in a test directory located in $HOME/easybuildinstall"
), None, 'store_true', False, 'p'),
'set-gid-bit': ("Set group ID bit on newly created directories",
None, 'store_true', False),
'sticky-bit': ("Set sticky bit on newly created directories", None,
'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true',
False, 't'),
'umask':
("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
None, 'store', None),
'update-modules-tool-cache':
("Update modules tool cache file(s) after generating module file",
None, 'store_true', False),
})
self.log.debug("override_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
def configure_step(self):
"""Custom configuration procedure for ALADIN."""
# unset $LIBRARY_PATH set by modules of dependencies, because it may screw up linking
if 'LIBRARY_PATH' in os.environ:
self.log.debug("Unsetting $LIBRARY_PATH (was: %s)" %
os.environ['LIBRARY_PATH'])
self.orig_library_path = os.environ.pop('LIBRARY_PATH')
# build auxiliary libraries
auxlibs_dir = None
my_gnu = None
if self.toolchain.comp_family() == toolchain.GCC:
my_gnu = 'y' # gfortran
for var in ['CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS']:
flags = os.getenv(var)
env.setvar(var,
"%s -fdefault-real-8 -fdefault-double-8" % flags)
self.log.info("Updated %s to '%s'" % (var, os.getenv(var)))
elif self.toolchain.comp_family() == toolchain.INTELCOMP:
my_gnu = 'i' # icc/ifort
else:
raise EasyBuildError(
"Don't know how to set 'my_gnu' variable in auxlibs build script."
)
self.log.info("my_gnu set to '%s'" % my_gnu)
tmp_installroot = tempfile.mkdtemp(prefix='aladin_auxlibs_')
try:
cwd = os.getcwd()
os.chdir(self.builddir)
builddirs = os.listdir(self.builddir)
auxlibs_dir = [
x for x in builddirs if x.startswith('auxlibs_installer')
][0]
os.chdir(auxlibs_dir)
auto_driver = 'driver_automatic'
for line in fileinput.input(auto_driver,
inplace=1,
backup='.orig.eb'):
line = re.sub(r"^(my_gnu\s*=\s*).*$", r"\1%s" % my_gnu, line)
line = re.sub(r"^(my_r32\s*=\s*).*$", r"\1n",
line) # always 64-bit real precision
line = re.sub(r"^(my_readonly\s*=\s*).*$", r"\1y",
line) # make libs read-only after build
line = re.sub(r"^(my_installroot\s*=\s*).*$",
r"\1%s" % tmp_installroot, line)
sys.stdout.write(line)
run_cmd("./%s" % auto_driver)
os.chdir(cwd)
except OSError as err:
raise EasyBuildError("Failed to build ALADIN: %s", err)
# build gmkpack, update PATH and set GMKROOT
# we build gmkpack here because a config file is generated in the gmkpack isntall path
try:
gmkpack_dir = [x for x in builddirs if x.startswith('gmkpack')][0]
os.chdir(os.path.join(self.builddir, gmkpack_dir))
qa = {
'Do you want to run the configuration file maker assistant now (y) or later [n] ?':
'n',
}
run_cmd_qa("./build_gmkpack", qa)
os.chdir(cwd)
paths = os.getenv('PATH').split(':')
paths.append(os.path.join(self.builddir, gmkpack_dir, 'util'))
env.setvar('PATH', ':'.join(paths))
env.setvar('GMKROOT', os.path.join(self.builddir, gmkpack_dir))
except OSError as err:
raise EasyBuildError("Failed to build gmkpack: %s", err)
# generate gmkpack configuration file
self.conf_file = 'ALADIN_%s' % self.version
self.conf_filepath = os.path.join(self.builddir, 'gmkpack_support',
'arch', '%s.x' % self.conf_file)
try:
if os.path.exists(self.conf_filepath):
os.remove(self.conf_filepath)
self.log.info("Removed existing gmpack config file %s" %
self.conf_filepath)
archdir = os.path.dirname(self.conf_filepath)
if not os.path.exists(archdir):
mkdir(archdir, parents=True)
except OSError as err:
raise EasyBuildError("Failed to remove existing file %s: %s",
self.conf_filepath, err)
mpich = 'n'
known_mpi_libs = [
toolchain.MPICH, toolchain.MPICH2, toolchain.INTELMPI
]
if self.toolchain.options.get(
'usempi',
None) and self.toolchain.mpi_family() in known_mpi_libs:
mpich = 'y'
qpref = 'Please type the ABSOLUTE name of '
qsuff = ', or ignore (environment variables allowed) :'
qsuff2 = ', or ignore : (environment variables allowed) :'
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.GCC:
gribdir = 'GNU'
elif comp_fam == toolchain.INTELCOMP:
gribdir = 'INTEL'
else:
raise EasyBuildError(
"Don't know which grib lib dir to use for compiler %s",
comp_fam)
aux_lib_gribex = os.path.join(tmp_installroot, gribdir, 'lib',
'libgribex.a')
aux_lib_ibm = os.path.join(tmp_installroot, gribdir, 'lib',
'libibmdummy.a')
grib_api_lib = os.path.join(get_software_root('grib_api'), 'lib',
'libgrib_api.a')
grib_api_f90_lib = os.path.join(get_software_root('grib_api'), 'lib',
'libgrib_api_f90.a')
grib_api_inc = os.path.join(get_software_root('grib_api'), 'include')
jasperlib = os.path.join(get_software_root('JasPer'), 'lib',
'libjasper.a')
mpilib = os.path.join(os.getenv('MPI_LIB_DIR'),
os.getenv('MPI_LIB_SHARED'))
# netCDF
netcdf = get_software_root('netCDF')
netcdf_fortran = get_software_root('netCDF-Fortran')
if netcdf:
netcdfinc = os.path.join(netcdf, 'include')
if netcdf_fortran:
netcdflib = os.path.join(netcdf_fortran,
get_software_libdir('netCDF-Fortran'),
'libnetcdff.a')
else:
netcdflib = os.path.join(netcdf, get_software_libdir('netCDF'),
'libnetcdff.a')
if not os.path.exists(netcdflib):
raise EasyBuildError("%s does not exist", netcdflib)
else:
raise EasyBuildError("netCDF(-Fortran) not available")
ldpaths = [ldflag[2:] for ldflag in os.getenv('LDFLAGS').split(' ')
] # LDFLAGS have form '-L/path/to'
lapacklibs = []
for lib in os.getenv('LAPACK_STATIC_LIBS').split(','):
libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths]
lapacklibs.append([
libpath for libpath in libpaths if os.path.exists(libpath)
][0])
lapacklib = ' '.join(lapacklibs)
blaslibs = []
for lib in os.getenv('BLAS_STATIC_LIBS').split(','):
libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths]
blaslibs.append([
libpath for libpath in libpaths if os.path.exists(libpath)
][0])
blaslib = ' '.join(blaslibs)
qa = {
'Do you want to run the configuration file maker assistant now (y) or later [n] ?':
'y',
'Do you want to setup your configuration file for MPICH (y/n) [n] ?':
mpich,
'Please type the directory name where to find a dummy file mpif.h or ignore :':
os.getenv('MPI_INC_DIR'),
'%sthe library gribex or emos%s' % (qpref, qsuff2):
aux_lib_gribex,
'%sthe library ibm%s' % (qpref, qsuff):
aux_lib_ibm,
'%sthe library grib_api%s' % (qpref, qsuff):
grib_api_lib,
'%sthe library grib_api_f90%s' % (qpref, qsuff):
grib_api_f90_lib,
'%sthe JPEG auxilary library if enabled by Grib_api%s' % (qpref, qsuff2):
jasperlib,
'%sthe library netcdf%s' % (qpref, qsuff):
netcdflib,
'%sthe library lapack%s' % (qpref, qsuff):
lapacklib,
'%sthe library blas%s' % (qpref, qsuff):
blaslib,
'%sthe library mpi%s' % (qpref, qsuff):
mpilib,
'%sa MPI dummy library for serial executions, or ignore :' % qpref:
'',
'Please type the directory name where to find grib_api headers, or ignore :':
grib_api_inc,
'Please type the directory name where to find fortint.h or ignore :':
'',
'Please type the directory name where to find netcdf headers, or ignore :':
netcdfinc,
'Do you want to define CANARI (y/n) [y] ?':
'y',
'Please type the name of the script file used to generate a preprocessed blacklist file, or ignore :':
'',
'Please type the name of the script file used to recover local libraries (gget), or ignore :':
'',
'Please type the options to tune the gnu compilers, or ignore :':
os.getenv('F90FLAGS'),
}
f90_seq = os.getenv('F90_SEQ')
if not f90_seq:
# F90_SEQ is only defined when usempi is enabled
f90_seq = os.getenv('F90')
stdqa = OrderedDict([
(r'Confirm library .* is .*',
'y'), # this one needs to be tried first!
(r'.*fortran 90 compiler name .*\s*:\n\(suggestions\s*: .*\)',
f90_seq),
(r'.*fortran 90 compiler interfaced with .*\s*:\n\(suggestions\s*: .*\)',
f90_seq),
(r'Please type the ABSOLUTE name of .*library.*, or ignore\s*[:]*\s*[\n]*.*',
''),
(r'Please .* to save this draft configuration file :\n.*',
'%s.x' % self.conf_file),
])
no_qa = [
".*ignored.",
]
env.setvar('GMKTMP', self.builddir)
env.setvar('GMKFILE', self.conf_file)
run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa)
# set environment variables for installation dirs
env.setvar('ROOTPACK', os.path.join(self.installdir, 'rootpack'))
env.setvar('ROOTBIN', os.path.join(self.installdir, 'rootpack'))
env.setvar('HOMEPACK', os.path.join(self.installdir, 'pack'))
env.setvar('HOMEBIN', os.path.join(self.installdir, 'pack'))
# patch config file to include right Fortran compiler flags
regex_subs = [(r"^(FRTFLAGS\s*=.*)$", r"\1 %s" % os.getenv('FFLAGS'))]
apply_regex_substitutions(self.conf_filepath, regex_subs)
def config_options(self):
# config options
descr = ("Configuration options", "Configure EasyBuild behavior.")
oldstyle_defaults = get_default_oldstyle_configfile_defaults()
opts = OrderedDict({
'avail-module-naming-schemes': ("Show all supported module naming schemes",
None, 'store_true', False,),
'avail-modules-tools': ("Show all supported module tools",
None, "store_true", False,),
'avail-repositories': ("Show all repository types (incl. non-usable)",
None, "store_true", False,),
'buildpath': ("Temporary build path", None, 'store', oldstyle_defaults['buildpath']),
'ignore-dirs': ("Directory names to ignore when searching for files/dirs",
'strlist', 'store', ['.git', '.svn']),
'installpath': ("Install path for software and modules", None, 'store', oldstyle_defaults['installpath']),
'config': ("Path to EasyBuild config file",
None, 'store', oldstyle_defaults['config'], 'C'),
'logfile-format': ("Directory name and format of the log file",
'strtuple', 'store', oldstyle_defaults['logfile_format'], {'metavar': 'DIR,FORMAT'}),
'module-naming-scheme': ("Module naming scheme",
'choice', 'store', oldstyle_defaults['module_naming_scheme'],
sorted(avail_module_naming_schemes().keys())),
'moduleclasses': (("Extend supported module classes "
"(For more info on the default classes, use --show-default-moduleclasses)"),
None, 'extend', oldstyle_defaults['moduleclasses']),
'modules-footer': ("Path to file containing footer to be added to all generated module files",
None, 'store_or_None', None, {'metavar': "PATH"}),
'modules-tool': ("Modules tool to use",
'choice', 'store', oldstyle_defaults['modules_tool'],
sorted(avail_modules_tools().keys())),
'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath "
"(repositorypath prefix is only relevant in case of FileRepository repository) "
"(used prefix for defaults %s)" % oldstyle_defaults['prefix']),
None, 'store', None),
'recursive-module-unload': ("Enable generating of modules that unload recursively.",
None, 'store_true', False),
'repository': ("Repository type, using repositorypath",
'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())),
'repositorypath': (("Repository path, used by repository "
"(is passed as list of arguments to create the repository instance). "
"For more info, use --avail-repositories."),
'strlist', 'store',
oldstyle_defaults['repositorypath'][oldstyle_defaults['repository']]),
'show-default-moduleclasses': ("Show default module classes with description",
None, 'store_true', False),
'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)",
None, 'store', oldstyle_defaults['sourcepath']),
'subdir-modules': ("Installpath subdir for modules", None, 'store', oldstyle_defaults['subdir_modules']),
'subdir-software': ("Installpath subdir for software", None, 'store', oldstyle_defaults['subdir_software']),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
'testoutput': ("Path to where a job should place the output (to be set within jobscript)",
None, 'store', None),
'tmp-logdir': ("Log directory where temporary log files are stored",
None, 'store', oldstyle_defaults['tmp_logdir']),
'tmpdir': ('Directory to use for temporary storage', None, 'store', None),
})
self.log.debug("config_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
