def apply_patch(patch_file, dest, fn=None, copy=False, level=None):
"""
Apply a patch to source code in directory dest
- assume unified diff created with "diff -ru old new"
"""
if build_option('extended_dry_run'):
# skip checking of files in dry run mode
patch_filename = os.path.basename(patch_file)
dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent'))
elif not os.path.isfile(patch_file):
raise EasyBuildError("Can't find patch %s: no such file", patch_file)
elif fn and not os.path.isfile(fn):
raise EasyBuildError("Can't patch file %s: no such file", fn)
elif not os.path.isdir(dest):
raise EasyBuildError("Can't patch directory %s: no such directory", dest)
# copy missing files
if copy:
if build_option('extended_dry_run'):
dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent'))
else:
try:
shutil.copy2(patch_file, dest)
_log.debug("Copied patch %s to dir %s" % (patch_file, dest))
# early exit, work is done after copying
return True
except IOError, err:
raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err)
def update_pr(pr, paths, commit_msg=None):
"""Update specified pull request using specified files."""
_log.experimental("Updating pull request #%s with %s", pr, paths)
github_user = build_option('github_user')
if github_user is None:
raise EasyBuildError("GitHub user must be specified to use --update-pr")
pr_target_account = build_option('pr_target_account')
pr_target_repo = build_option('pr_target_repo')
pr_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr]
status, pr_data = github_api_get_request(pr_url, github_user)
if not status == HTTP_STATUS_OK:
raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
pr, pr_target_account, pr_target_repo, status, pr_data)
# branch that corresponds with PR is supplied in form <account>:<branch_label>
account = pr_data['head']['label'].split(':')[0]
branch = ':'.join(pr_data['head']['label'].split(':')[1:])
github_target = '%s/%s' % (pr_target_account, pr_target_repo)
print_msg("Determined branch name corresponding to %s PR #%s: %s" % (github_target, pr, branch), log=_log)
_, _, _, diff_stat = _easyconfigs_pr_common(paths, start_branch=branch, pr_branch=branch,
target_account=account, commit_msg=commit_msg)
print_msg("Overview of changes:\n%s\n" % diff_stat, log=_log, prefix=False)
full_repo = '%s/%s' % (pr_target_account, pr_target_repo)
msg = "Updated %s PR #%s by pushing to branch %s/%s" % (full_repo, pr, account, branch)
if build_option('dry_run') or build_option('extended_dry_run'):
msg += " [DRY RUN]"
print_msg(msg, log=_log, prefix=False)
def apply_regex_substitutions(path, regex_subs):
"""
Apply specified list of regex substitutions.
@param path: path to file to patch
@param regex_subs: list of substitutions to apply, specified as (<regexp pattern>, <replacement string>)
"""
# only report when in 'dry run' mode
if build_option('extended_dry_run'):
dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent'))
for regex, subtxt in regex_subs:
dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt))
else:
_log.debug("Applying following regex substitutions to %s: %s", path, regex_subs)
for i, (regex, subtxt) in enumerate(regex_subs):
regex_subs[i] = (re.compile(regex), subtxt)
try:
for line in fileinput.input(path, inplace=1, backup='.orig.eb'):
for regex, subtxt in regex_subs:
line = regex.sub(subtxt, line)
sys.stdout.write(line)
except OSError, err:
raise EasyBuildError("Failed to patch %s: %s", path, err)
def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
:param unload_module: name(s) of module to unload first
"""
body = []
if unload_modules:
body.extend([self.unload_module(m).strip() for m in unload_modules])
load_template = self.LOAD_TEMPLATE
# Lmod 7.6.1+ supports depends-on which does this most nicely:
if build_option('mod_depends_on') or depends_on:
if not modules_tool().supports_depends_on:
raise EasyBuildError("depends-on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
body.append(load_template)
if build_option('recursive_mod_unload') or recursive_unload or load_template == self.LOAD_TEMPLATE_DEPENDS_ON:
# not wrapping the 'module load' with an is-loaded guard ensures recursive unloading;
# when "module unload" is called on the module in which the dependency "module load" is present,
# it will get translated to "module unload"
load_statement = body + ['']
else:
load_statement = [self.conditional_statement("is-loaded %(mod_name)s", '\n'.join(body), negative=True)]
return '\n'.join([''] + load_statement) % {'mod_name': mod_name}
def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state):
"""
Upload/dump overall test report
@param ecs_with_res: processed easyconfigs with build result (success/failure)
@param orig_cnt: number of original easyconfig paths
@param success: boolean indicating whether all builds were successful
@param msg: message to be included in test report
@param init_session_state: initial session state info to include in test report
"""
dump_path = build_option('dump_test_report')
pr_nr = build_option('from_pr')
upload = build_option('upload_test_report')
if upload:
msg = msg + " (%d easyconfigs in this PR)" % orig_cnt
test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True)
if pr_nr:
# upload test report to gist and issue a comment in the PR to notify
txt = post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success)
else:
# only upload test report as a gist
gist_url = upload_test_report_as_gist(test_report)
txt = "Test report uploaded to %s" % gist_url
else:
test_report = create_test_report(msg, ecs_with_res, init_session_state)
txt = None
_log.debug("Test report: %s" % test_report)
if dump_path is not None:
write_file(dump_path, test_report)
_log.info("Test report dumped to %s" % dump_path)
return txt
def init(self):
"""
Initialise the GC3Pie job backend.
"""
# List of config files for GC3Pie; non-existing ones will be
# silently ignored. The list here copies GC3Pie's default,
# for the principle of minimal surprise, but there is no
# strict requirement that this be done and EB could actually
# choose to use a completely distinct set of conf. files.
self.config_files = gc3libs.Default.CONFIG_FILE_LOCATIONS[:]
cfgfile = build_option('job_backend_config')
if cfgfile:
self.config_files.append(cfgfile)
self.output_dir = build_option('job_output_dir')
self.job_cnt = 0
job_deps_type = build_option('job_deps_type')
if job_deps_type is None:
job_deps_type = JOB_DEPS_TYPE_ABORT_ON_ERROR
self.log.info("Using default job dependency type: %s", job_deps_type)
else:
self.log.info("Using specified job dependency type: %s", job_deps_type)
if job_deps_type == JOB_DEPS_TYPE_ALWAYS_RUN:
self.jobs = DependentTaskCollection(output_dir=self.output_dir)
elif job_deps_type == JOB_DEPS_TYPE_ABORT_ON_ERROR:
self.jobs = AbortingDependentTaskCollection(output_dir=self.output_dir)
else:
raise EasyBuildError("Unknown job dependency type specified: %s", job_deps_type)
# after polling for job status, sleep for this time duration
# before polling again (in seconds)
self.poll_interval = build_option('job_polling_interval')
def det_easyconfig_paths(orig_paths):
"""
Determine paths to easyconfig files.
@param orig_paths: list of original easyconfig paths
@return: list of paths to easyconfig files
"""
from_pr = build_option('from_pr')
robot_path = build_option('robot_path')
# list of specified easyconfig files
ec_files = orig_paths[:]
if from_pr is not None:
pr_files = fetch_easyconfigs_from_pr(from_pr)
if ec_files:
# replace paths for specified easyconfigs that are touched in PR
for i, ec_file in enumerate(ec_files):
for pr_file in pr_files:
if ec_file == os.path.basename(pr_file):
ec_files[i] = pr_file
else:
# if no easyconfigs are specified, use all the ones touched in the PR
ec_files = [path for path in pr_files if path.endswith('.eb')]
if ec_files and robot_path:
# look for easyconfigs with relative paths in robot search path,
# unless they were found at the given relative paths
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, ec_file in enumerate(ec_files):
if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file):
ecs_to_find.append((idx, ec_file))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths in robot search path
for path in robot_path:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
ec_files[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if not ecs_to_find:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')]
# stop os.walk insanity as soon as we have all we need (outer loop)
if not ecs_to_find:
break
return ec_files
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50):
"""
Run specified interactive command (in a subshell)
@param cmd: command to run
@param qa: dictionary which maps question to answers
@param no_qa: list of patters that are not questions
@param log_ok: only run output/exit code for failing commands (exit code non-zero)
@param log_all: always log command output and exit code
@param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
@param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
@param std_qa: dictionary which maps question regex patterns to answers
@param path: path to execute the command is; current working directory is used if unspecified
"""
cwd = os.getcwd()
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def parse_easyconfigs(paths, validate=True):
"""
Parse easyconfig files
:param paths: paths to easyconfigs
"""
easyconfigs = []
generated_ecs = False
for (path, generated) in paths:
path = os.path.abspath(path)
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
raise EasyBuildError("Can't find path %s", path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
for ec_file in ec_files:
kwargs = {'validate': validate}
# only pass build specs when not generating easyconfig files
if not build_option('try_to_generate'):
kwargs['build_specs'] = build_option('build_specs')
easyconfigs.extend(process_easyconfig(ec_file, **kwargs))
except IOError, err:
raise EasyBuildError("Processing easyconfigs in path %s failed: %s", path, err)
def test_strict(self):
"""Test use of --strict."""
# check default
self.assertEqual(build_option('strict'), run.WARN)
for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]:
options = init_config(args=['--strict=%s' % strict_str])
init_config(build_options={'strict': options.strict})
self.assertEqual(build_option('strict'), strict_val)
def __init__(self, name=None, version=None, mns=None, class_constants=None, tcdeps=None, modtool=None,
hidden=False):
"""
Toolchain constructor.
:param name: toolchain name
:param version: toolchain version
:param mns: module naming scheme to use
:param class_constants: toolchain 'constants' to define
:param tcdeps: list of toolchain 'dependencies' (i.e., the toolchain components)
:param modtool: ModulesTool instance to use
:param hidden: bool indicating whether toolchain is hidden or not
"""
self.base_init()
self.dependencies = []
self.toolchain_dep_mods = []
self.cached_compilers = set()
if name is None:
name = self.NAME
if name is None:
raise EasyBuildError("Toolchain init: no name provided")
self.name = name
if version is None:
version = self.VERSION
if version is None:
raise EasyBuildError("Toolchain init: no version provided")
self.version = version
self.modules = []
self.vars = None
self._init_class_constants(class_constants)
self.tcdeps = tcdeps
# toolchain instances are created before initiating build options sometimes, e.g. for --list-toolchains
self.dry_run = build_option('extended_dry_run', default=False)
hidden_toolchains = build_option('hide_toolchains', default=None) or []
self.hidden = hidden or (name in hidden_toolchains)
self.modules_tool = modtool
self.use_rpath = False
self.mns = mns
self.mod_full_name = None
self.mod_short_name = None
self.init_modpaths = None
if self.name != DUMMY_TOOLCHAIN_NAME:
# sometimes no module naming scheme class instance can/will be provided, e.g. with --list-toolchains
if self.mns is not None:
tc_dict = self.as_dict()
self.mod_full_name = self.mns.det_full_module_name(tc_dict)
self.mod_short_name = self.mns.det_short_module_name(tc_dict)
self.init_modpaths = self.mns.det_init_modulepaths(tc_dict)
def complete(self):
"""
Complete a bulk job submission.
Create engine, and progress it until all jobs have terminated.
"""
# create an instance of `Engine` using the list of configuration files
try:
self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True)
except gc3libs.exceptions.Error as err:
raise EasyBuildError("Failed to create GC3Pie engine: %s", err)
# make sure that all job log files end up in the same directory, rather than renaming the output directory
# see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine
self._engine.retrieve_overwrites = True
# some sites may not be happy with flooding the cluster with build jobs...
self._engine.max_in_flight = build_option('job_max_jobs')
# `Engine.stats()` (which is used later on in `_print_status_report()`)
# changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible
# with both
try:
self._engine.init_stats_for(Application)
except AttributeError:
_log.debug("No `init_stats_for` method in the Engine class;"
" assuming pre-2.5.0 GC3Pie and ignoring error.")
# Add your application to the engine. This will NOT submit
# your application yet, but will make the engine *aware* of
# the application.
self._engine.add(self.jobs)
# in case you want to select a specific resource, call
target_resource = build_option('job_target_resource')
if target_resource:
res = self._engine.select_resource(target_resource)
if res == 0:
raise EasyBuildError("Failed to select target resource '%s' in GC3Pie", target_resource)
# Periodically check the status of your application.
while self.jobs.execution.state != Run.State.TERMINATED:
# `Engine.progress()` will do the GC3Pie magic:
# submit new jobs, update status of submitted jobs, get
# results of terminating jobs etc...
self._engine.progress()
# report progress
self._print_status_report()
# Wait a few seconds...
time.sleep(self.poll_interval)
# final status report
print_msg("Done processing jobs", log=self.log, silent=build_option('silent'))
self._print_status_report()
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
maxhits=50, trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def dry_run(easyconfigs, modtool, short=False):
"""
Compose dry run overview for supplied easyconfigs:
* [ ] for unavailable
* [x] for available
* [F] for forced
* [R] for rebuild
:param easyconfigs: list of parsed easyconfigs (EasyConfig instances)
:param modtool: ModulesTool instance to use
:param short: use short format for overview: use a variable for common prefixes
"""
lines = []
if build_option('robot_path') is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
lines.append("Dry run: printing build status of easyconfigs and dependencies")
all_specs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)
unbuilt_specs = skip_available(all_specs, modtool)
dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front)
listed_ec_paths = [spec['spec'] for spec in easyconfigs]
var_name = 'CFGS'
common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs if spec['spec'] is not None])
# only allow short if common prefix is long enough
short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2
for spec in all_specs:
if spec in unbuilt_specs:
ans = ' '
elif build_option('force') and spec['spec'] in listed_ec_paths:
ans = 'F'
elif build_option('rebuild') and spec['spec'] in listed_ec_paths:
ans = 'R'
else:
ans = 'x'
if spec['ec'] is not None and spec['ec'].short_mod_name != spec['ec'].full_mod_name:
mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name)
else:
mod = spec['full_mod_name']
if spec['spec'] is None:
item = "(no easyconfig file found)"
elif short:
item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:])
else:
item = spec['spec']
lines.append(dry_run_fmt % (ans, item, mod))
if short:
# insert after 'Dry run:' message
lines.insert(1, "%s=%s" % (var_name, common_prefix))
return '\n'.join(lines)
def __init__(self, easyconfigs):
self.container_base = build_option('container_base')
self.container_build_image = build_option('container_build_image')
self.container_path = container_path()
self.easyconfigs = easyconfigs
self.image_format = build_option('container_image_format')
self.img_name = build_option('container_image_name')
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.mns = ActiveMNS()
self.tmpdir = build_option('container_tmpdir')
def search_easyconfigs(query, short=False):
"""Search for easyconfigs, if a query is provided."""
robot_path = build_option("robot_path")
if robot_path:
search_path = robot_path
else:
search_path = [os.getcwd()]
ignore_dirs = build_option("ignore_dirs")
silent = build_option("silent")
search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent)
def _set_compiler_flags(self):
"""Collect the flags set, and add them as variables too"""
flags = [self.options.option(x) for x in self.COMPILER_FLAGS if self.options.get(x, False)]
cflags = [self.options.option(x) for x in self.COMPILER_C_FLAGS + self.COMPILER_C_UNIQUE_FLAGS \
if self.options.get(x, False)]
fflags = [self.options.option(x) for x in self.COMPILER_F_FLAGS + self.COMPILER_F_UNIQUE_FLAGS \
if self.options.get(x, False)]
# Allow a user-defined default optimisation
default_opt_level = build_option('default_opt_level')
if default_opt_level not in self.COMPILER_OPT_FLAGS:
raise EasyBuildError("Unknown value for default optimisation: %s (possibilities are %s)" %
(default_opt_level, self.COMPILER_OPT_FLAGS))
# 1st one is the one to use. add default at the end so len is at least 1
optflags = ([self.options.option(x) for x in self.COMPILER_OPT_FLAGS if self.options.get(x, False)] + \
[self.options.option(default_opt_level)])[:1]
# only apply if the vectorize toolchainopt is explicitly set
# otherwise the individual compiler toolchain file should make sure that
# vectorization is disabled for noopt and lowopt, and enabled otherwise.
if self.options.get('vectorize') is not None:
vectoptions = self.options.option('vectorize')
vectflags = vectoptions[self.options['vectorize']]
# avoid double use of such flags, or e.g. -fno-tree-vectorize followed by -ftree-vectorize
if isinstance(optflags[0], list):
optflags[0] = [flag for flag in optflags[0] if flag not in vectoptions.values()]
optflags.append(vectflags)
optarchflags = []
if build_option('optarch') == OPTARCH_GENERIC:
# don't take 'optarch' toolchain option into account when --optarch=GENERIC is used,
# *always* include the flags that correspond to generic compilation (which are listed in 'optarch' option)
optarchflags.append(self.options.option('optarch'))
elif self.options.get('optarch', False):
optarchflags.append(self.options.option('optarch'))
precflags = [self.options.option(x) for x in self.COMPILER_PREC_FLAGS if self.options.get(x, False)] + \
[self.options.option('defaultprec')]
self.variables.nextend('OPTFLAGS', optflags + optarchflags)
self.variables.nextend('PRECFLAGS', precflags[:1])
# precflags last
for var in ['CFLAGS', 'CXXFLAGS']:
self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS')
self.variables.nextend(var, flags)
self.variables.nextend(var, cflags)
for var in ['FCFLAGS', 'FFLAGS', 'F90FLAGS']:
self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS')
self.variables.nextend(var, flags)
self.variables.nextend(var, fflags)
def search_easyconfigs(query, short=False, filename_only=False, terse=False):
"""Search for easyconfigs, if a query is provided."""
robot_path = build_option('robot_path')
if robot_path:
search_path = robot_path
else:
search_path = [os.getcwd()]
ignore_dirs = build_option('ignore_dirs')
silent = build_option('silent')
search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent, filename_only=filename_only,
terse=terse)
def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'):
"""
Creates a job to build a *single* easyconfig.
:param job_backend: A factory object for querying server parameters and creating actual job objects
:param build_command: format string for command, full path to an easyconfig file will be substituted in it
:param easyconfig: easyconfig as processed by process_easyconfig
:param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
returns the job
"""
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
for env_var in ["PYTHONPATH", "MODULEPATH"]:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# determine whether additional options need to be passed to the 'eb' command
add_opts = ''
if easyconfig['hidden']:
add_opts += ' --hidden'
# create command based on build_command template
command = build_command % {
'add_opts': add_opts,
'output_dir': os.path.join(os.path.abspath(output_dir), name),
'spec': easyconfig['spec'],
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
extra = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
extra['hours'] = int(math.ceil(previous_time * 2 / 60))
if build_option('job_cores'):
extra['cores'] = build_option('job_cores')
job = job_backend.make_job(command, name, easybuild_vars, **extra)
job.module = easyconfig['ec'].full_mod_name
return job
def dry_run(easyconfigs, modtool, short=False):
"""
Compose dry run overview for supplied easyconfigs:
* [ ] for unavailable
* [x] for available
* [F] for forced
* [R] for rebuild
@param easyconfigs: list of parsed easyconfigs (EasyConfig instances)
@param modtool: ModulesTool instance to use
@param short: use short format for overview: use a variable for common prefixes
"""
lines = []
if build_option("robot_path") is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
lines.append("Dry run: printing build status of easyconfigs and dependencies")
all_specs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)
unbuilt_specs = skip_available(all_specs, modtool)
dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front)
listed_ec_paths = [spec["spec"] for spec in easyconfigs]
var_name = "CFGS"
common_prefix = det_common_path_prefix([spec["spec"] for spec in all_specs])
# only allow short if common prefix is long enough
short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2
for spec in all_specs:
if spec in unbuilt_specs:
ans = " "
elif build_option("force") and spec["spec"] in listed_ec_paths:
ans = "F"
elif build_option("rebuild") and spec["spec"] in listed_ec_paths:
ans = "R"
else:
ans = "x"
if spec["ec"].short_mod_name != spec["ec"].full_mod_name:
mod = "%s | %s" % (spec["ec"].mod_subdir, spec["ec"].short_mod_name)
else:
mod = spec["ec"].full_mod_name
if short:
item = os.path.join("$%s" % var_name, spec["spec"][len(common_prefix) + 1 :])
else:
item = spec["spec"]
lines.append(dry_run_fmt % (ans, item, mod))
if short:
# insert after 'Dry run:' message
lines.insert(1, "%s=%s" % (var_name, common_prefix))
return "\n".join(lines)
def search_easyconfigs(query, short=False, filename_only=False, terse=False):
"""Search for easyconfigs, if a query is provided."""
search_path = build_option("robot_path")
if not search_path:
search_path = [os.getcwd()]
ignore_dirs = build_option("ignore_dirs")
# note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs
var_defs, _hits = search_file(
search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse, silent=True, filename_only=False
)
# filter out archived easyconfigs, these are handled separately
hits, archived_hits = [], []
for hit in _hits:
if EASYCONFIGS_ARCHIVE_DIR in hit.split(os.path.sep):
archived_hits.append(hit)
else:
hits.append(hit)
# check whether only filenames should be printed
if filename_only:
hits = [os.path.basename(hit) for hit in hits]
archived_hits = [os.path.basename(hit) for hit in archived_hits]
# prepare output format
if terse:
lines, tmpl = [], "%s"
else:
lines = ["%s=%s" % var_def for var_def in var_defs]
tmpl = " * %s"
# non-archived hits are shown first
lines.extend(tmpl % hit for hit in hits)
# also take into account archived hits
if archived_hits:
if build_option("consider_archived_easyconfigs"):
if not terse:
lines.extend(["", "Matching archived easyconfigs:", ""])
lines.extend(tmpl % hit for hit in archived_hits)
elif not terse:
cnt = len(archived_hits)
lines.extend(
[
"",
"Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them"
% cnt,
]
)
print "\n".join(lines)
def build_singularity_image(def_path):
"""Build Singularity container image by calling out to 'singularity' (requires admin privileges!)."""
cont_path = container_path()
def_file = os.path.basename(def_path)
# use --imagename if specified, otherwise derive based on filename of recipe
img_name = build_option('container_image_name')
if img_name is None:
# definition file Singularity.<app>-<version, container name <app>-<version>.<img|simg>
img_name = def_file.split('.', 1)[1]
cmd_opts = ''
image_format = build_option('container_image_format')
# squashfs image format (default for Singularity)
if image_format in [None, CONT_IMAGE_FORMAT_SQUASHFS]:
img_path = os.path.join(cont_path, img_name + '.simg')
# ext3 image format, creating as writable container
elif image_format == CONT_IMAGE_FORMAT_EXT3:
img_path = os.path.join(cont_path, img_name + '.img')
cmd_opts = '--writable'
# sandbox image format, creates as a directory but acts like a container
elif image_format == CONT_IMAGE_FORMAT_SANDBOX:
img_path = os.path.join(cont_path, img_name)
cmd_opts = '--sandbox'
else:
raise EasyBuildError("Unknown container image format specified for Singularity: %s" % image_format)
if os.path.exists(img_path):
if build_option('force'):
print_msg("WARNING: overwriting existing container image at %s due to --force" % img_path)
remove_file(img_path)
else:
raise EasyBuildError("Container image already exists at %s, not overwriting it without --force", img_path)
# resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo...
singularity = which('singularity')
cmd_env = ''
singularity_tmpdir = build_option('container_tmpdir')
if singularity_tmpdir:
cmd_env += 'SINGULARITY_TMPDIR=%s' % singularity_tmpdir
cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, def_path])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd)
run_cmd(cmd, stream_output=True)
print_msg("Singularity image created at %s" % img_path, log=_log)
def _set_fftw_variables(self):
if not hasattr(self, 'BLAS_LIB_DIR'):
raise EasyBuildError("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)")
imklver = get_software_version(self.FFT_MODULE_NAME[0])
picsuff = ''
if self.options.get('pic', None):
picsuff = '_pic'
bitsuff = '_lp64'
if self.options.get('i8', None):
bitsuff = '_ilp64'
compsuff = '_intel'
if get_software_root('icc') is None:
if get_software_root('GCC'):
compsuff = '_gnu'
else:
raise EasyBuildError("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.")
fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)]
if self.options['usempi']:
# add cluster interface for recent imkl versions
if LooseVersion(imklver) >= LooseVersion("11.0.2"):
fftw_libs.append("fftw3x_cdft%s%s" % (bitsuff, picsuff))
elif LooseVersion(imklver) >= LooseVersion("10.3"):
fftw_libs.append("fftw3x_cdft%s" % picsuff)
fftw_libs.append("mkl_cdft_core") # add cluster dft
fftw_libs.extend(self.variables['LIBBLACS'].flatten()) # add BLACS; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
fftw_libs.extend(self.variables['LIBBLAS'].flatten()) # add BLAS libs (contains dft)
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
self.FFT_LIB_DIR = self.BLAS_LIB_DIR
self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR
# building the FFTW interfaces is optional,
# so make sure libraries are there before FFT_LIB is set
imklroot = get_software_root(self.FFT_MODULE_NAME[0])
fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
# filter out gfortran from list of FFTW libraries to check for, since it's not provided by imkl
check_fftw_libs = [lib for lib in fftw_libs if lib != 'gfortran']
fftw_lib_exists = lambda x: any([os.path.exists(os.path.join(d, "lib%s.a" % x)) for d in fft_lib_dirs])
if all([fftw_lib_exists(lib) for lib in check_fftw_libs]):
self.FFT_LIB = fftw_libs
else:
msg = "Not all FFTW interface libraries %s are found in %s" % (check_fftw_libs, fft_lib_dirs)
msg += ", can't set $FFT_LIB."
if build_option('extended_dry_run'):
dry_run_warning(msg, silent=build_option('silent'))
else:
raise EasyBuildError(msg)
def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
- load modules for toolchain and dependencies
- generate extra variables and set them in the environment
:param onlymod: boolean/string to indicate if the toolchain should only load the environment
with module (True) or also set all other variables (False) like compiler CC etc
(If string: comma separated list of variables that will be ignored).
:param silent: keep quiet, or not (mostly relates to extended dry run output)
:param loadmod: whether or not to (re)load the toolchain module, and the modules for the dependencies
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
:param rpath_include_dirs: extra directories to include in RPATH
"""
if loadmod:
self._load_modules(silent=silent)
if self.name != DUMMY_TOOLCHAIN_NAME:
trace_msg("defining build environment for %s/%s toolchain" % (self.name, self.version))
if not self.dry_run:
self._verify_toolchain()
# Generate the variables to be set
self.set_variables()
# set the variables
# onlymod can be comma-separated string of variables not to be set
if onlymod == True:
self.log.debug("prepare: do not set additional variables onlymod=%s", onlymod)
self.generate_vars()
else:
self.log.debug("prepare: set additional variables onlymod=%s", onlymod)
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod, verbose=not silent)
# consider f90cache first, since ccache can also wrap Fortran compilers
for cache_tool in [F90CACHE, CCACHE]:
if build_option('use_%s' % cache_tool):
self.prepare_compiler_cache(cache_tool)
if build_option('rpath'):
if self.options.get('rpath', True):
self.prepare_rpath_wrappers(rpath_filter_dirs, rpath_include_dirs)
self.use_rpath = True
else:
self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option")
def singularity(easyconfigs, container_base=None):
"""
Create Singularity definition file and (optionally) image
"""
check_singularity()
if container_base is None:
container_base = build_option('container_base')
def_path = generate_singularity_recipe(easyconfigs, container_base)
# also build container image, if requested (requires sudo!)
if build_option('container_build_image'):
build_singularity_image(def_path)
def regtest(easyconfig_paths, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
@param easyconfig_paths: path of easyconfigs to run regtest on
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place
# all log files and the test output as xml
basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path']
regtest_output_dir = build_option('regtest_output_dir')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif var in os.environ:
output_dir = os.path.abspath(os.environ[var])
else:
# default: current dir + easybuild-test-[timestamp]
output_dir = os.path.join(cur_dir, basename)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
else:
_log.error("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
def write_file(path, txt, append=False, forced=False):
"""Write given contents to file at given path (overwrites current file contents!)."""
# early exit in 'dry run' mode
if not forced and build_option('extended_dry_run'):
dry_run_msg("file written: %s" % path, silent=build_option('silent'))
return
# note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
try:
mkdir(os.path.dirname(path), parents=True)
with open(path, 'a' if append else 'w') as handle:
handle.write(txt)
except IOError, err:
raise EasyBuildError("Failed to write to %s: %s", path, err)
def mkdir(path, parents=False, set_gid=None, sticky=None):
"""
Create a directory
Directory is the path to create
@param parents: create parent directories if needed (mkdir -p)
@param set_gid: set group ID bit, to make subdirectories and files inherit group
@param sticky: set the sticky bit on this directory (a.k.a. the restricted deletion flag),
to avoid users can removing/renaming files in this directory
"""
if set_gid is None:
set_gid = build_option('set_gid_bit')
if sticky is None:
sticky = build_option('sticky_bit')
if not os.path.isabs(path):
path = os.path.abspath(path)
# exit early if path already exists
if not os.path.exists(path):
tup = (path, parents, set_gid, sticky)
_log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)" % tup)
# set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path
existing_parent_path = os.path.dirname(path)
try:
if parents:
# climb up until we hit an existing path or the empty string (for relative paths)
while existing_parent_path and not os.path.exists(existing_parent_path):
existing_parent_path = os.path.dirname(existing_parent_path)
os.makedirs(path)
else:
os.mkdir(path)
except OSError, err:
_log.error("Failed to create directory %s: %s" % (path, err))
# set group ID and sticky bits, if desired
bits = 0
if set_gid:
bits |= stat.S_ISGID
if sticky:
bits |= stat.S_ISVTX
if bits:
try:
new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep)
new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0])
adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True)
except OSError, err:
_log.error("Failed to set groud ID/sticky bit: %s" % err)
def define_env_var(self, env_var):
"""
Determine whether environment variable with specified name should be defined or not.
:param env_var: name of environment variable to check
"""
return env_var not in (build_option('filter_env_vars') or [])
def __init__(self, mself, ext):
"""
mself has the logger
"""
self.master = mself
self.log = self.master.log
self.cfg = self.master.cfg.copy()
self.ext = copy.deepcopy(ext)
self.dry_run = self.master.dry_run
if not 'name' in self.ext:
raise EasyBuildError("'name' is missing in supplied class instance 'ext'.")
# parent sanity check paths/commands are not relevant for extension
self.cfg['sanity_check_commands'] = []
self.cfg['sanity_check_paths'] = []
# list of source/patch files: we use an empty list as default value like in EasyBlock
self.src = self.ext.get('src', [])
self.patches = self.ext.get('patches', [])
self.options = copy.deepcopy(self.ext.get('options', {}))
# don't re-prepare the build environment when doing a dry run, since it'll be the same as for the parent
if not build_option('extended_dry_run'):
self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True)
self.sanity_check_fail_msgs = []
def test_step(self):
"""
Custom test procedure for Molpro.
Run 'make quicktest, make test', but only for source install and if license is available.
"""
# Only bother to check if the licence token is available
if os.path.isfile(
self.license_token) and not self.cfg['precompiled_binaries']:
# check 'main routes' only
run_cmd("make quicktest")
if build_option('mpi_tests'):
# extensive test
run_cmd("make MOLPRO_OPTIONS='-n%s' test" %
self.cfg['parallel'])
else:
self.log.info(
"Skipping extensive testing of Molpro since MPI testing is disabled"
)
def check_module_path(self):
"""
Check if MODULEPATH is set and change it if necessary.
"""
# if self.mod_paths is not specified, define it and make sure the EasyBuild module path is in there (first)
if self.mod_paths is None:
# take (unique) module paths from environment
self.set_mod_paths()
self.log.debug("self.mod_paths set based on $MODULEPATH: %s" % self.mod_paths)
# determine module path for EasyBuild install path to be included in $MODULEPATH
eb_modpath = os.path.join(install_path(typ='modules'), build_option('suffix_modules_path'))
# make sure EasyBuild module path is in 1st place
self.prepend_module_path(eb_modpath)
self.log.info("Prepended list of module paths with path used by EasyBuild: %s" % eb_modpath)
# set the module path environment accordingly
for mod_path in self.mod_paths[::-1]:
self.use(mod_path)
self.log.info("$MODULEPATH set based on list of module paths (via 'module use'): %s" % os.environ['MODULEPATH'])
def test_step(self):
"""Run GAMESS-US tests (if 'runtest' easyconfig parameter is set to True)."""
# don't use provided 'runall' script for tests, since that only runs the tests single-core
if self.cfg['runtest']:
if not build_option('mpi_tests'):
self.log.info("Skipping testing of GAMESS-US since MPI testing is disabled")
return
try:
cwd = os.getcwd()
os.chdir(self.testdir)
except OSError, err:
raise EasyBuildError("Failed to move to temporary directory for running tests: %s", err)
# copy input files for exam<id> standard tests
for test_input in glob.glob(os.path.join(self.installdir, 'tests', 'standard', 'exam*.inp')):
try:
shutil.copy2(test_input, os.getcwd())
except OSError, err:
raise EasyBuildError("Failed to copy %s to %s: %s", test_input, os.getcwd(), err)
def prepare(self):
"""
Creates the absolute filename for the module.
"""
mod_path_suffix = build_option('suffix_modules_path')
full_mod_name = self.app.full_mod_name
# module file goes in general moduleclass category
self.filename = os.path.join(self.module_path, mod_path_suffix, full_mod_name)
# make symlink in moduleclass category
mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.app.cfg)
self.class_mod_files = [os.path.join(self.module_path, p, full_mod_name) for p in mod_symlink_paths]
# create directories and links
for path in [os.path.dirname(x) for x in [self.filename] + self.class_mod_files]:
mkdir(path, parents=True)
# remove module file if it's there (it'll be recreated), see Application.make_module
if os.path.exists(self.filename):
os.remove(self.filename)
return os.path.join(self.module_path, mod_path_suffix)
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def test_step(self):
"""Custom built-in test procedure for HPCG."""
if self.cfg['runtest']:
if not build_option('mpi_tests'):
self.log.info(
"Skipping testing of HPCG since MPI testing is disabled")
return
objbindir = os.path.join(self.cfg['start_dir'], 'obj', 'bin')
# obtain equivalent of 'mpirun -np 2 xhpcg'
hpcg_mpi_cmd = self.toolchain.mpi_cmd_for("xhpcg", 2)
# 2 threads per MPI process (4 threads in total)
cmd = "PATH=%s:$PATH OMP_NUM_THREADS=2 %s" % (objbindir,
hpcg_mpi_cmd)
run_cmd(cmd, simple=True, log_all=True, log_ok=True)
# find log file, check for success
success_regex = re.compile(r"Scaled Residual \[[0-9.e-]+\]")
try:
hpcg_logs = glob.glob('hpcg_log*txt')
if len(hpcg_logs) == 1:
txt = open(hpcg_logs[0], 'r').read()
self.log.debug("Contents of HPCG log file %s: %s" %
(hpcg_logs[0], txt))
if success_regex.search(txt):
self.log.info(
"Found pattern '%s' in HPCG log file %s, OK!",
success_regex.pattern, hpcg_logs[0])
else:
raise EasyBuildError(
"Failed to find pattern '%s' in HPCG log file %s",
success_regex.pattern, hpcg_logs[0])
else:
raise EasyBuildError(
"Failed to find exactly one HPCG log file: %s",
hpcg_logs)
except OSError, err:
raise EasyBuildError(
"Failed to check for success in HPCG log file: %s", err)
def build_step(self, *args, **kwargs):
"""Custom build procedure for Python, ensure stack size limit is set to 'unlimited' (if desired)."""
# make sure installation directory doesn't already exist when building with --rpath and
# configuring with --enable-optimizations, since that leads to errors like:
# ./python: symbol lookup error: ./python: undefined symbol: __gcov_indirect_call
# see also https://bugs.python.org/issue29712
enable_opts_flag = '--enable-optimizations'
if build_option('rpath') and enable_opts_flag in self.cfg['configopts']:
if os.path.exists(self.installdir):
warning_msg = "Removing existing installation directory '%s', "
warning_msg += "because EasyBuild is configured to use RPATH linking "
warning_msg += "and %s configure option is used." % enable_opts_flag
print_warning(warning_msg % self.installdir)
remove_dir(self.installdir)
if self.cfg['ulimit_unlimited']:
# determine current stack size limit
(out, _) = run_cmd("ulimit -s")
curr_ulimit_s = out.strip()
# figure out hard limit for stack size limit;
# this determines whether or not we can use "ulimit -s unlimited"
(out, _) = run_cmd("ulimit -s -H")
max_ulimit_s = out.strip()
if curr_ulimit_s == UNLIMITED:
self.log.info("Current stack size limit is %s: OK", curr_ulimit_s)
elif max_ulimit_s == UNLIMITED:
self.log.info("Current stack size limit is %s, setting it to %s for build...",
curr_ulimit_s, UNLIMITED)
self.cfg.update('prebuildopts', "ulimit -s %s && " % UNLIMITED)
else:
msg = "Current stack size limit is %s, and can not be set to %s due to hard limit of %s;"
msg += " setting stack size limit to %s instead, "
msg += " this may break part of the compilation (e.g. hashlib)..."
print_warning(msg % (curr_ulimit_s, UNLIMITED, max_ulimit_s, max_ulimit_s))
self.cfg.update('prebuildopts', "ulimit -s %s && " % max_ulimit_s)
super(EB_Python, self).build_step(*args, **kwargs)
def build_step(self):
"""Build and install WRF and testcases using provided compile script."""
# enable parallel build
par = self.cfg['parallel']
self.par = ''
if par:
self.par = "-j %s" % par
# fix compile script shebang to use provided tcsh
cmpscript = os.path.join(self.start_dir, 'compile')
tcsh_root = get_software_root('tcsh')
if tcsh_root:
tcsh_path = os.path.join(tcsh_root, 'bin', 'tcsh')
# avoid using full path to tcsh if possible, since it may be too long to be used as shebang line
which_tcsh = which('tcsh')
if which_tcsh and os.path.samefile(which_tcsh, tcsh_path):
env_path = os.path.join('/usr', 'bin', 'env')
# use env command from alternate sysroot, if available
sysroot = build_option('sysroot')
if sysroot:
sysroot_env_path = os.path.join(sysroot, 'usr', 'bin',
'env')
if os.path.exists(sysroot_env_path):
env_path = sysroot_env_path
new_shebang = env_path + ' tcsh'
else:
new_shebang = tcsh_path
regex_subs = [('^#!/bin/csh.*', '#!' + new_shebang)]
apply_regex_substitutions(cmpscript, regex_subs)
# build wrf
cmd = "%s %s wrf" % (cmpscript, self.par)
run_cmd(cmd, log_all=True, simple=True, log_output=True)
# build two testcases to produce ideal.exe and real.exe
for test in ["em_real", "em_b_wave"]:
cmd = "%s %s %s" % (cmpscript, self.par, test)
run_cmd(cmd, log_all=True, simple=True, log_output=True)
def _set_compiler_flags(self):
"""Collect the flags set, and add them as variables too"""
flags = [
self.options.option(x) for x in self.COMPILER_FLAGS
if self.options.get(x, False)
]
cflags = [self.options.option(x) for x in self.COMPILER_C_FLAGS + self.COMPILER_C_UNIQUE_FLAGS \
if self.options.get(x, False)]
fflags = [self.options.option(x) for x in self.COMPILER_F_FLAGS + self.COMPILER_F_UNIQUE_FLAGS \
if self.options.get(x, False)]
# 1st one is the one to use. add default at the end so len is at least 1
optflags = [self.options.option(x) for x in self.COMPILER_OPT_FLAGS if self.options.get(x, False)] + \
[self.options.option('defaultopt')]
optarchflags = []
if build_option('optarch') == OPTARCH_GENERIC:
# don't take 'optarch' toolchain option into account when --optarch=GENERIC is used,
# *always* include the flags that correspond to generic compilation (which are listed in 'optarch' option)
optarchflags.append(self.options.option('optarch'))
elif self.options.get('optarch', False):
optarchflags.append(self.options.option('optarch'))
precflags = [self.options.option(x) for x in self.COMPILER_PREC_FLAGS if self.options.get(x, False)] + \
[self.options.option('defaultprec')]
self.variables.nextend('OPTFLAGS', optflags[:1] + optarchflags)
self.variables.nextend('PRECFLAGS', precflags[:1])
# precflags last
for var in ['CFLAGS', 'CXXFLAGS']:
self.variables.nappend(var, flags)
self.variables.nappend(var, cflags)
self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS')
for var in ['FCFLAGS', 'FFLAGS', 'F90FLAGS']:
self.variables.nappend(var, flags)
self.variables.nappend(var, fflags)
self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS')
def github_api_get_request(request_f, github_user=None, token=None, **kwargs):
"""
Helper method, for performing get requests to GitHub API.
:param request_f: function that should be called to compose request, providing a RestClient instance
:param github_user: GitHub user name (to try and obtain matching GitHub token if none is provided)
:param token: GitHub token to use
@return: tuple with return status and data
"""
if github_user is None:
github_user = build_option('github_user')
if token is None:
token = fetch_github_token(github_user)
url = request_f(
RestClient(GITHUB_API_URL, username=github_user, token=token))
try:
status, data = url.get(**kwargs)
except socket.gaierror, err:
_log.warning("Error occured while performing get request: %s" % err)
status, data = 0, None
def __init__(self, mod_paths=None):
"""
Create a ModulesTool object
@param mod_paths: A list of paths where the modules can be located
@type mod_paths: list
"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.mod_paths = None
if mod_paths is not None:
self.set_mod_paths(mod_paths)
# DEPRECATED!
self._modules = []
# actual module command (i.e., not the 'module' wrapper function, but the binary)
self.cmd = self.COMMAND
if self.COMMAND_ENVIRONMENT is not None and self.COMMAND_ENVIRONMENT in os.environ:
self.log.debug('Set command via environment variable %s' %
self.COMMAND_ENVIRONMENT)
self.cmd = os.environ[self.COMMAND_ENVIRONMENT]
if self.cmd is None:
self.log.error('No command set.')
else:
self.log.debug('Using command %s' % self.cmd)
# version of modules tool
self.version = None
# some initialisation/verification
self.check_cmd_avail()
self.check_module_path()
self.check_module_function(
allow_mismatch=build_option('allow_modules_tool_mismatch'))
self.set_and_check_version()
# this can/should be set to True during testing
self.testing = False
def find_starccm_subdirs(self):
"""Determine subdirectory of install directory in which STAR-CCM+ was installed."""
cwd = change_dir(self.installdir)
cands = glob.glob(
os.path.join(self.version + '*', 'STAR-CCM+%s*' % self.version))
if len(cands) == 1:
self.starccm_subdir = cands[0]
self.log.info("Found STAR-CCM+ subdirectory: %s",
self.starccm_subdir)
self.starview_subdir = os.path.join(
os.path.dirname(self.starccm_subdir),
'STAR-View+%s' % self.version)
elif self.dry_run or build_option('module_only'):
# Provide fake string values if executing a dry-run or module-only build
self.starccm_subdir = ''
self.starview_subdir = ''
else:
raise EasyBuildError(
"Failed to determine the STAR-CCM+ subdirectory in %s: %s",
self.installdir, cands)
change_dir(cwd)
def load_module(self, mod_name, recursive_unload=False, unload_modules=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
:param unload_module: name(s) of module to unload first
"""
body = []
if unload_modules:
body.extend([self.unload_module(m).strip() for m in unload_modules])
body.append(self.LOAD_TEMPLATE)
if build_option('recursive_mod_unload') or recursive_unload:
# not wrapping the 'module load' with an is-loaded guard ensures recursive unloading;
# when "module unload" is called on the module in which the dependency "module load" is present,
# it will get translated to "module unload"
load_statement = body + ['']
else:
load_statement = [self.conditional_statement("is-loaded %(mod_name)s", '\n'.join(body), negative=True)]
return '\n'.join([''] + load_statement) % {'mod_name': mod_name}
def run_hook(label,
hooks,
pre_step_hook=False,
post_step_hook=False,
args=None,
msg=None):
"""
Run hook with specified label and return result of calling the hook or None.
:param label: name of hook
:param hooks: dict of defined hooks
:param pre_step_hook: indicates whether hook to run is a pre-step hook
:param post_step_hook: indicates whether hook to run is a post-step hook
:param args: arguments to pass to hook function
:param msg: custom message that is printed when hook is called
"""
hook = find_hook(label,
hooks,
pre_step_hook=pre_step_hook,
post_step_hook=post_step_hook)
res = None
if hook:
if args is None:
args = []
if pre_step_hook:
label = 'pre-' + label
elif post_step_hook:
label = 'post-' + label
if msg is None:
msg = "Running %s hook..." % label
if build_option('debug'):
print_msg(msg)
_log.info("Running '%s' hook function (arguments: %s)...",
hook.__name__, args)
res = hook(*args)
return res
def test_use_rich_show_progress_bars(self):
"""Test use_rich and show_progress_bar functions."""
# restore default configuration to show progress bars (disabled to avoid mangled test output)
update_build_option('show_progress_bar', True)
self.assertEqual(build_option('output_style'), 'auto')
if HAVE_RICH:
self.assertTrue(use_rich())
self.assertTrue(show_progress_bars())
update_build_option('output_style', 'rich')
self.assertTrue(use_rich())
self.assertTrue(show_progress_bars())
else:
self.assertFalse(use_rich())
self.assertFalse(show_progress_bars())
update_build_option('output_style', 'basic')
self.assertFalse(use_rich())
self.assertFalse(show_progress_bars())
def dependencies(self):
"""
Returns an array of parsed dependencies (after filtering, if requested)
dependency = {'name': '', 'version': '', 'dummy': (False|True), 'versionsuffix': '', 'toolchain': ''}
"""
deps = self['dependencies'] + self.builddependencies()
# if filter-deps option is provided we "clean" the list of dependencies for
# each processed easyconfig to remove the unwanted dependencies
filter_deps = build_option('filter_deps')
if filter_deps:
self.log.debug("Dependencies BEFORE filtering: %s" % deps)
filtered_deps = []
for dep in deps:
if dep['name'] not in filter_deps:
filtered_deps.append(dep)
else:
self.log.info("filtered out dependency %s" % dep)
self.log.debug("Dependencies AFTER filtering: %s" % filtered_deps)
deps = filtered_deps
return deps
def check_singularity():
"""Check whether Singularity can be used (if it's needed)."""
# if we're going to build a container image, we'll need a sufficiently recent version of Singularity available
# (and otherwise we don't really care if Singularity is not available)
if build_option('container_build_image'):
path_to_singularity_cmd = which('singularity')
if path_to_singularity_cmd:
print_msg("Singularity tool found at %s" % path_to_singularity_cmd)
out, ec = run_cmd("singularity --version", simple=False, trace=False, force_in_dry_run=True)
if ec:
raise EasyBuildError("Failed to determine Singularity version: %s" % out)
else:
# singularity version format for 2.3.1 and higher is x.y-dist
singularity_version = out.strip().split('-')[0]
if LooseVersion(singularity_version) < LooseVersion('2.4'):
raise EasyBuildError("Please upgrade singularity instance to version 2.4 or higher")
else:
print_msg("Singularity version '%s' is 2.4 or higher ... OK" % singularity_version)
else:
raise EasyBuildError("Singularity not found in your system")
def __init__(self, *args, **kwargs):
"""Initialisation of custom class variables for ELPA."""
super(EB_ELPA, self).__init__(*args, **kwargs)
for flag in ELPA_CPU_FEATURE_FLAGS:
# fail-safe: make sure we're not overwriting an existing attribute (could lead to weird bugs if we do)
if hasattr(self, flag):
raise EasyBuildError("EasyBlock attribute '%s' already exists")
setattr(self, flag, self.cfg['use_%s' % flag])
# auto-detect CPU features that can be used and are not enabled/disabled explicitly,
# but only if --optarch=GENERIC is not being used
if self.cfg['auto_detect_cpu_features']:
# if --optarch=GENERIC is used, we will not use no CPU feature
if build_option('optarch') == OPTARCH_GENERIC:
cpu_features = []
else:
cpu_features = ELPA_CPU_FEATURE_FLAGS
self.log.info("CPU features considered for auto-detection: %s",
cpu_features)
# get list of available CPU features, so we can check which ones to retain
avail_cpu_features = get_cpu_features()
# on macOS, AVX is indicated with 'avx1.0' rather than 'avx'
if 'avx1.0' in avail_cpu_features:
avail_cpu_features.append('avx')
self.log.info("List of available CPU features: %s",
avail_cpu_features)
for flag in cpu_features:
# only enable use of a particular CPU feature if it's still undecided (i.e. None)
if getattr(self, flag) is None and flag in avail_cpu_features:
self.log.info(
"Enabling use of %s (should be supported based on CPU features)",
flag.upper())
setattr(self, flag, True)
def prepare_compiler_cache(self, cache_tool):
"""
Prepare for using specified compiler caching tool (e.g., ccache, f90cache)
:param cache_tool: name of compiler caching tool to prepare for
"""
compilers = self.comp_cache_compilers(cache_tool)
self.log.debug("Using compiler cache tool '%s' for compilers: %s", cache_tool, compilers)
# set paths that should be used by compiler caching tool
comp_cache_path = build_option('use_%s' % cache_tool)
setvar('%s_DIR' % cache_tool.upper(), comp_cache_path)
setvar('%s_TEMPDIR' % cache_tool.upper(), tempfile.mkdtemp())
cache_path = which(cache_tool)
if cache_path is None:
raise EasyBuildError("%s binary not found in $PATH, required by --use-compiler-cache", cache)
else:
self.symlink_commands({cache_tool: (cache_path, compilers)})
self.cached_compilers.update(compilers)
self.log.debug("Cached compilers (after preparing for %s): %s", cache_tool, self.cached_compilers)
def check_python_version():
"""Check currently used Python version."""
python_maj_ver = sys.version_info[0]
python_min_ver = sys.version_info[1]
python_ver = '%d.%d' % (python_maj_ver, python_min_ver)
_log.info("Found Python version %s", python_ver)
silence_deprecation_warnings = build_option(
'silence_deprecation_warnings') or []
if python_maj_ver == 2:
if python_min_ver < 6:
raise EasyBuildError(
"Python 2.6 or higher is required when using Python 2, found Python %s",
python_ver)
elif python_min_ver == 6:
depr_msg = "Running EasyBuild with Python 2.6 is deprecated"
if 'Python26' in silence_deprecation_warnings:
_log.warning(depr_msg)
else:
_log.deprecated(depr_msg, '5.0')
else:
_log.info("Running EasyBuild with Python 2 (version %s)",
python_ver)
elif python_maj_ver == 3:
if python_min_ver < 5:
raise EasyBuildError(
"Python 3.5 or higher is required when using Python 3, found Python %s",
python_ver)
else:
_log.info("Running EasyBuild with Python 3 (version %s)",
python_ver)
else:
raise EasyBuildError(
"EasyBuild is not compatible (yet) with Python %s", python_ver)
return (python_maj_ver, python_min_ver)
def get_gromacs_arch(self):
"""Determine value of GMX_SIMD CMake flag based on optarch string.
Refs:
[0] http://manual.gromacs.org/documentation/2016.3/install-guide/index.html#typical-installation
[1] http://manual.gromacs.org/documentation/2016.3/install-guide/index.html#simd-support
[2] http://www.gromacs.org/Documentation/Acceleration_and_parallelization
"""
# default: fall back on autodetection
res = None
optarch = build_option('optarch') or ''
# take into account that optarch value is a dictionary if it is specified by compiler family
if isinstance(optarch, dict):
comp_fam = self.toolchain.comp_family()
optarch = optarch.get(comp_fam, '')
optarch = optarch.upper()
if 'MIC-AVX512' in optarch and LooseVersion(self.version) >= LooseVersion('2016'):
res = 'AVX_512_KNL'
elif 'AVX512' in optarch and LooseVersion(self.version) >= LooseVersion('2016'):
res = 'AVX_512'
elif 'AVX2' in optarch and LooseVersion(self.version) >= LooseVersion('5.0'):
res = 'AVX2_256'
elif 'AVX' in optarch:
res = 'AVX_256'
elif 'SSE3' in optarch or 'SSE2' in optarch or 'MARCH=NOCONA' in optarch:
# Gromacs doesn't have any GMX_SIMD=SSE3 but only SSE2 and SSE4.1 [1].
# According to [2] the performance difference between SSE2 and SSE4.1 is minor on x86
# and SSE4.1 is not supported by AMD Magny-Cours[1].
res = 'SSE2'
if res:
self.log.info("Target architecture based on optarch configuration option ('%s'): %s", optarch, res)
else:
self.log.info("No target architecture specified based on optarch configuration option ('%s')", optarch)
return res
def det_full_module_name(ec, eb_ns=False):
"""
Determine full module name following the currently active module naming scheme.
First try to pass 'parsed' easyconfig as supplied,
try and find a matching easyconfig file, parse it and supply it in case of a KeyError.
"""
try:
mod_name = _det_full_module_name(ec, eb_ns=eb_ns)
except KeyError, err:
_log.debug(
"KeyError '%s' when determining module name for %s, trying fallback procedure..."
% (err, ec))
# for dependencies, only name/version/versionsuffix/toolchain easyconfig parameters are available;
# when a key error occurs, try and find an easyconfig file to parse via the robot,
# and retry with the parsed easyconfig file (which will contains a full set of keys)
robot = build_option('robot_path')
eb_file = robot_find_easyconfig(robot, ec['name'],
det_full_ec_version(ec))
if eb_file is None:
_log.error(
"Failed to find an easyconfig file when determining module name for: %s"
% ec)
else:
parsed_ec = process_easyconfig(eb_file)
if len(parsed_ec) > 1:
_log.warning(
"More than one parsed easyconfig obtained from %s, only retaining first"
% eb_file)
try:
mod_name = _det_full_module_name(parsed_ec[0]['ec'],
eb_ns=eb_ns)
except KeyError, err:
_log.error(
"A KeyError '%s' occured when determining a module name for %s."
% parsed_ec['ec'])
def match_minimum_tc_specs(source_tc_spec, target_tc_hierarchy):
"""
Match a source toolchain spec to the minimal corresponding toolchain in a target hierarchy
:param source_tc_spec: specs of source toolchain
:param target_tc_hierarchy: hierarchy of specs for target toolchain
"""
minimal_matching_toolchain = {}
target_compiler_family = ''
# break out once we've found the first match since the hierarchy is ordered low to high in terms of capabilities
for target_tc_spec in target_tc_hierarchy:
if check_capability_mapping(source_tc_spec, target_tc_spec):
# GCCcore has compiler capabilities,
# but should only be used in the target if the original toolchain was also GCCcore
if target_tc_spec['name'] != GCCcore.NAME or source_tc_spec[
'name'] == GCCcore.NAME:
minimal_matching_toolchain = {
'name': target_tc_spec['name'],
'version': target_tc_spec['version']
}
target_compiler_family = target_tc_spec['comp_family']
break
if not minimal_matching_toolchain:
raise EasyBuildError(
"No possible mapping from source toolchain spec %s to target toolchain hierarchy specs %s",
source_tc_spec, target_tc_hierarchy)
# Warn if we are changing compiler families, this is very likely to cause problems
if target_compiler_family != source_tc_spec['comp_family']:
print_warning(
"Your request will result in a compiler family switch (%s to %s). Here be dragons!"
% (source_tc_spec['comp_family'], target_compiler_family),
silent=build_option('silent'))
return minimal_matching_toolchain
def test_step(self):
"""Run NAMD test case."""
if self.cfg['runtest']:
if not build_option('mpi_tests'):
self.log.info(
"Skipping testing of NAMD since MPI testing is disabled")
return
namdcmd = os.path.join(self.cfg['start_dir'], self.namd_arch,
'namd%s' % self.version.split('.')[0])
if self.cfg['charm_arch'].startswith('mpi'):
namdcmd = self.toolchain.mpi_cmd_for(namdcmd, 2)
ppn = ''
if self.toolchain.options.get('openmp', False):
ppn = '+ppn 2'
cmd = "%(namd)s %(ppn)s %(testdir)s" % {
'namd':
namdcmd,
'ppn':
ppn,
'testdir':
os.path.join(self.cfg['start_dir'], self.namd_arch, 'src',
'alanin'),
}
out, ec = run_cmd(cmd, simple=False)
if ec == 0:
test_ok_regex = re.compile(
"(^Program finished.$|End of program\s*$)", re.M)
if test_ok_regex.search(out):
self.log.debug("Test '%s' ran fine." % cmd)
else:
raise EasyBuildError(
"Test '%s' failed ('%s' not found), output: %s", cmd,
test_ok_regex.pattern, out)
else:
self.log.debug("Skipping running NAMD test case after building")
def __init__(self, script, name, env_vars=None, hours=None, cores=None):
"""Create a new Job to be submitted to SLURM."""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.jobid = None
self.script = script
self.name = name
self.job_specs = {
'job-name': self.name,
# pattern for output file for submitted job;
# SLURM replaces %x with job name, %j with job ID (see https://slurm.schedmd.com/sbatch.html#lbAF)
'output': '%x-%j.out',
'wrap': self.script,
}
if env_vars:
self.job_specs['export'] = ','.join(sorted(env_vars.keys()))
max_walltime = build_option('job_max_walltime')
if hours is None:
hours = max_walltime
if hours > max_walltime:
self.log.warn(
"Specified %s hours, but this is impossible. (resetting to %s hours)"
% (hours, max_walltime))
hours = max_walltime
self.job_specs['time'] = hours * 60
if cores:
self.job_specs['nodes'] = 1
self.job_specs['ntasks'] = cores
self.job_specs['ntasks-per-node'] = cores
else:
self.log.warn(
"Number of cores to request not specified, falling back to whatever Slurm does by default"
)
def run(self, *args, **kwargs):
"""Perform the actual Python package build/installation procedure"""
if not self.src:
raise EasyBuildError(
"No source found for Python package %s, required for installation. (src: %s)",
self.name, self.src)
# we unpack unless explicitly told otherwise
kwargs.setdefault('unpack_src', self._should_unpack_source())
super(PythonPackage, self).run(*args, **kwargs)
# configure, build, test, install
# See EasyBlock.get_steps
steps = [
(CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step],
True),
(BUILD_STEP, 'building', [lambda x: x.build_step], True),
(TEST_STEP, 'testing', [lambda x: x._test_step], True),
(INSTALL_STEP, "installing", [lambda x: x.install_step], True),
]
self.skip = False # --skip does not apply here
self.silent = build_option('silent')
# See EasyBlock.run_all_steps
for (step_name, descr, step_methods, skippable) in steps:
if self.skip_step(step_name, skippable):
print_msg("\t%s [skipped]" % descr,
log=self.log,
silent=self.silent)
else:
if self.dry_run:
self.dry_run_msg("\t%s... [DRY RUN]\n", descr)
else:
print_msg("\t%s..." % descr,
log=self.log,
silent=self.silent)
for step_method in step_methods:
step_method(self)()
def update(self):
"""Update after new modules were added."""
if build_option('update_modules_tool_cache'):
spider_cmd = os.path.join(os.path.dirname(self.cmd), 'spider')
cmd = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']]
self.log.debug("Running command '%s'..." % ' '.join(cmd))
proc = subprocess.Popen(cmd,
stdout=PIPE,
stderr=PIPE,
env=os.environ)
(stdout, stderr) = proc.communicate()
if stderr:
raise EasyBuildError("An error occured when running '%s': %s",
' '.join(cmd), stderr)
if self.testing:
# don't actually update local cache when testing, just return the cache contents
return stdout
else:
try:
cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua')
self.log.debug(
"Updating Lmod spider cache %s with output from '%s'" %
(cache_fp, ' '.join(cmd)))
cache_dir = os.path.dirname(cache_fp)
if not os.path.exists(cache_dir):
mkdir(cache_dir, parents=True)
cache_file = open(cache_fp, 'w')
cache_file.write(stdout)
cache_file.close()
except (IOError, OSError), err:
raise EasyBuildError(
"Failed to update Lmod spider cache %s: %s", cache_fp,
err)
def test_step(self):
"""Run tests using each of the backends."""
if build_option('rpath'):
# inject build location of libflexiblas.so.3 to $LD_LIBRARY_PATH
# (which is used as a fallback to find libraries not found via the locations listed in the RPATH section),
# to ensure that test binaries can find the FlexiBLAS library
ld_library_path = ':'.join([
os.path.join(self.obj_builddir, 'lib'),
os.path.join(self.obj_builddir, 'lib64'),
'$LD_LIBRARY_PATH'
])
self.cfg['pretestopts'] = ('export LD_LIBRARY_PATH="%s" && ' % ld_library_path) + self.cfg['pretestopts']
# Show verbose output if a test fails
setvar('CTEST_OUTPUT_ON_FAILURE', 'True')
# run tests with default backend (NETLIB)
test_cmd = ' '.join([
self.cfg['pretestopts'],
"make test",
self.cfg['testopts'],
])
run_cmd(test_cmd)
def __init__(self, *args, **kwargs):
"""Constructor."""
# early check for required commands
for cmd in ['sbatch', 'scontrol']:
path = which(cmd)
if path is None:
raise EasyBuildError("Required command '%s' not found", cmd)
super(Slurm, self).__init__(*args, **kwargs)
job_deps_type = build_option('job_deps_type')
if job_deps_type is None:
job_deps_type = JOB_DEPS_TYPE_ABORT_ON_ERROR
self.log.info("Using default job dependency type: %s", job_deps_type)
else:
self.log.info("Using specified job dependency type: %s", job_deps_type)
if job_deps_type == JOB_DEPS_TYPE_ABORT_ON_ERROR:
self.job_deps_type = 'afterok'
elif job_deps_type == JOB_DEPS_TYPE_ALWAYS_RUN:
self.job_deps_type = 'afterany'
else:
raise EasyBuildError("Unknown job dependency type specified: %s", job_deps_type)
def robot_find_easyconfig(name, version):
"""
Find an easyconfig for module in path
"""
key = (name, version)
if key in _easyconfig_files_cache:
_log.debug("Obtained easyconfig path from cache for %s: %s" % (key, _easyconfig_files_cache[key]))
return _easyconfig_files_cache[key]
paths = build_option('robot_path')
if not isinstance(paths, (list, tuple)):
if paths is None:
_log.error("No robot path specified, which is required when looking for easyconfigs (use --robot)")
paths = [paths]
# candidate easyconfig paths
for path in paths:
easyconfigs_paths = create_paths(path, name, version)
for easyconfig_path in easyconfigs_paths:
_log.debug("Checking easyconfig path %s" % easyconfig_path)
if os.path.isfile(easyconfig_path):
_log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path))
_easyconfig_files_cache[key] = os.path.abspath(easyconfig_path)
return _easyconfig_files_cache[key]
return None