def cmdline_easyconfigs_style_check(ecs):
"""
Run easyconfigs style check of each of the specified easyconfigs, triggered from 'eb' command line
:param ecs: list of easyconfigs to check, could be either file paths or EasyConfig instances
:return: True when style check passed on all easyconfig files, False otherwise
"""
print_msg("\nRunning style check on %d easyconfig(s)...\n" % len(ecs), prefix=False)
style_check_passed = True
for ec in ecs:
# if an EasyConfig instance is provided, just grab the corresponding file path
if isinstance(ec, EasyConfig):
path = ec.path
elif isinstance(ec, string_type):
path = ec
else:
raise EasyBuildError("Value of unknown type encountered in cmdline_easyconfigs_style_check: %s (type: %s)",
ec, type(ec))
if check_easyconfigs_style([path]) == 0:
res = 'PASS'
else:
res = 'FAIL'
style_check_passed = False
print_msg('[%s] %s' % (res, path), prefix=False)
return style_check_passed
def configure_step(self):
"""Custom configuration procedure for Paraver: template configuration options before using them."""
component = self.components[self.current_component]
try:
os.chdir(component)
except OSError as err:
raise EasyBuildError("Failed to move to %s: %s" % (component, err))
self.log.info("Customized start directory for component %s: %s", component, os.getcwd())
print_msg("starting with component %s" % component, log=self.log)
# check for required Boost dependency
boost = get_software_root('Boost')
if not boost:
raise EasyBuildError("Boost is not available as a dependency")
if not self.wxpropgrid:
raise EasyBuildError("wxPropertyGrid is not available as a dependency")
self.cfg['configopts'] = self.cfg['configopts'] % {
'boost': boost,
'installdir': self.installdir,
'wxpropgrid': self.wxpropgrid,
}
super(EB_Paraver, self).configure_step()
def configure_step(self):
"""Custom configuration procedure for TAU: template configuration options before using them."""
if self.cc is None or self.cxx is None or self.fortran is None:
raise EasyBuildError("Compiler family not supported yet: %s", self.toolchain.comp_family())
if self.mpi_inc_dir is None or self.mpi_lib_dir is None:
raise EasyBuildError("Failed to determine MPI include/library paths, no MPI available in toolchain?")
# make sure selected default TAU makefile will be available
avail_makefiles = ['Makefile.' + l for l in self.variant_labels]
if self.cfg['tau_makefile'] not in avail_makefiles:
raise EasyBuildError("Specified tau_makefile %s will not be available (only: %s)",
self.cfg['tau_makefile'], avail_makefiles)
# inform which backend/variant is being handled
backend = (['tau'] + self.cfg['extra_backends'])[self.variant_index // 3]
variant = ['mpi', 'openmp', 'hybrid'][self.variant_index % 3]
print_msg("starting with %s backend (%s variant)" % (backend, variant), log=self.log, silent=self.silent)
self.cfg['configopts'] = self.cfg['configopts'] % {
'backend_opt': self.backend_opts[backend],
'cc': self.cc,
'cxx': self.cxx,
'fortran': self.fortran,
'mpi_inc_dir': self.mpi_inc_dir,
'mpi_lib_dir': self.mpi_lib_dir,
'opt_pkgs_opts': self.opt_pkgs_opts,
}
for key in ['preconfigopts', 'configopts', 'prebuildopts', 'preinstallopts']:
self.log.debug("%s for TAU (variant index: %s): %s", key, self.variant_index, self.cfg[key])
super(EB_TAU, self).configure_step()
self.variant_index += 1
def install_step(self):
"""Custom install procedure for Stata."""
change_dir(self.installdir)
cmd = os.path.join(self.cfg['start_dir'], 'install')
std_qa = {
r"Do you wish to continue\?\s*\(y/n or q to quit\)": 'y',
r"Are you sure you want to install into .*\?\s*\(y/n or q\)": 'y',
r"Okay to proceed\s*\(y/n or q to quit\)": 'y',
}
no_qa = [
"About to proceed with installation:",
"uncompressing files",
"extracting files",
"setting permissions",
]
run_cmd_qa(cmd, {},
no_qa=no_qa,
std_qa=std_qa,
log_all=True,
simple=True)
print_msg(
"Note: you need to manually run ./stinit in %s to initialise the license for Stata!",
self.installdir)
def configure_step(self):
"""
Go through each component and setup configuration for the later Bundle install step
"""
super(EB_Clang_minus_AOMP, self).configure_step()
# Ensure necessary libraries are downloaded and can be found
device_lib_dir_pattern = os.path.join(self.builddir,
'ROCm-Device-Libs-*')
hits = glob.glob(device_lib_dir_pattern)
if len(hits) == 1:
self.device_lib_path = hits[0]
else:
raise EasyBuildError(
"Could not find 'ROCm-Device-Libs' source directory in %s",
self.builddir)
num_comps = len(self.cfg['components'])
for idx, comp in enumerate(self.comp_cfgs):
name = comp['name']
msg = "configuring bundle component %s %s (%d/%d)..." % (
name, comp['version'], idx + 1, num_comps)
print_msg(msg)
if name in self.cfg_method:
self.cfg_method[name](comp)
self.log.info(msg)
else:
self.log.warn("Component %s has no configure method!" % name)
def sanity_check_step(self, *args, **kwargs):
"""
If component sanity checks are enabled, run sanity checks for the desired components listed.
If nothing is being installed, just being able to load the (fake) module is sufficient
"""
if self.cfg['exts_list'] or self.cfg['sanity_check_paths'] or self.cfg[
'sanity_check_commands']:
super(Bundle, self).sanity_check_step(*args, **kwargs)
else:
self.log.info(
"Testing loading of module '%s' by means of sanity check" %
self.full_mod_name)
fake_mod_data = self.load_fake_module(purge=True)
self.log.debug("Cleaning up after testing loading of module")
self.clean_up_fake_module(fake_mod_data)
# run sanity checks for specific components
cnt = len(self.comp_cfgs_sanity_check)
for idx, comp in enumerate(self.comp_cfgs_sanity_check):
comp_name, comp_ver = comp.cfg['name'], comp.cfg['version']
print_msg("sanity checking bundle component %s v%s (%i/%i)...",
comp_name, comp_ver, idx + 1, cnt)
self.log.info("Starting sanity check step for component %s v%s",
comp_name, comp_ver)
comp.run_step('sanity_check', [lambda x: x.sanity_check_step])
def init_repo(path, repo_name, silent=False):
"""
Initialize a new Git repository at the specified location.
@param path: location where Git repository should be initialized
@param repo_name: name of Git repository
@param silent: keep quiet (don't print any messages)
"""
repo_path = os.path.join(path, repo_name)
# copy or init git working directory
git_working_dirs_path = build_option('git_working_dirs_path')
if git_working_dirs_path:
workdir = os.path.join(git_working_dirs_path, repo_name)
if os.path.exists(workdir):
try:
print_msg("copying %s..." % workdir, silent=silent)
shutil.copytree(workdir, repo_path)
except OSError as err:
raise EasyBuildError("Failed to copy git working dir %s to %s: %s", workdir, repo_path, err)
if not os.path.exists(repo_path):
mkdir(repo_path, parents=True)
try:
repo = git.Repo.init(repo_path)
except GitCommandError as err:
raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err)
_log.debug("temporary git working directory ready at %s", repo_path)
return repo
def cleanup_logfile_and_exit(logfile, testing, doexit):
"""Cleanup the logfile and exit"""
if not testing and logfile is not None:
os.remove(logfile)
print_msg("temporary log file %s has been removed." % (logfile), log=None, silent=testing)
if doexit:
sys.exit(0)
def obtain_path(specs, paths, try_to_generate=False, exit_on_error=True, silent=False):
"""Obtain a path for an easyconfig that matches the given specifications."""
# if no easyconfig files/paths were provided, but we did get a software name,
# we can try and find a suitable easyconfig ourselves, or generate one if we can
(generated, fn) = easyconfig.tools.obtain_ec_for(specs, paths, None)
if not generated:
return (fn, generated)
else:
# if an easyconfig was generated, make sure we're allowed to use it
if try_to_generate:
print_msg("Generated an easyconfig file %s, going to use it now..." % fn, silent=silent)
return (fn, generated)
else:
try:
os.remove(fn)
except OSError, err:
print_warning("Failed to remove generated easyconfig file %s." % fn)
print_error(
(
"Unable to find an easyconfig for the given specifications: %s; "
"to make EasyBuild try to generate a matching easyconfig, "
"use the --try-X options "
)
% specs,
log=_log,
exit_on_error=exit_on_error,
)
def check_tool(tool_name, min_tool_version=None):
"""
This function is a predicate check for the existence of tool_name on the system PATH.
If min_tool_version is not None, it will check that the version has an equal or higher value.
"""
tool_path = which(tool_name)
if not tool_path:
return False
print_msg("{0} tool found at {1}".format(tool_name, tool_path))
if not min_tool_version:
return True
version_cmd = "{0} --version".format(tool_name)
out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True)
if ec:
raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out))
res = re.search("\d+\.\d+(\.\d+)?", out.strip())
if not res:
raise EasyBuildError("Error parsing version for tool {0}".format(tool_name))
tool_version = res.group(0)
version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version)
if version_ok:
print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version))
return version_ok
def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
"""
Run hook with specified label.
:param label: name of hook
:param hooks: list of defined hooks
:param pre_step_hook: indicates whether hook to run is a pre-step hook
:param post_step_hook: indicates whether hook to run is a post-step hook
:param args: arguments to pass to hook function
:param msg: custom message that is printed when hook is called
"""
hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
if hook:
if args is None:
args = []
if pre_step_hook:
label = 'pre-' + label
elif post_step_hook:
label = 'post-' + label
if msg is None:
msg = "Running %s hook..." % label
print_msg(msg)
_log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
hook(*args)
def complete(self):
"""
Complete a bulk job submission.
Release all user holds on submitted jobs, and disconnect from server.
"""
for job in self._submitted:
if job.has_holds():
self.log.info("releasing user hold on job %s" % job.jobid)
job.release_hold()
self.disconnect_from_server()
# print list of submitted jobs
submitted_jobs = '; '.join([
"%s (%s): %s" % (job.name, job.module, job.jobid)
for job in self._submitted
])
print_msg("List of submitted jobs (%d): %s" %
(len(self._submitted), submitted_jobs),
log=self.log)
# determine leaf nodes in dependency graph, and report them
all_deps = set()
for job in self._submitted:
all_deps = all_deps.union(job.deps)
leaf_nodes = []
for job in self._submitted:
if job.jobid not in all_deps:
leaf_nodes.append(str(job.jobid).split('.')[0])
self.log.info("Job ids of leaf nodes in dep. graph: %s" %
','.join(leaf_nodes))
def build_and_install_software(module, options, origEnviron, exitOnFailure=True, silent=False):
"""
Build the software
"""
spec = module['spec']
print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
# restore original environment
_log.info("Resetting environment")
filetools.errorsFoundInLog = 0
modify_env(os.environ, origEnviron)
cwd = os.getcwd()
# load easyblock
easyblock = options.easyblock
if not easyblock:
# try to look in .eb file
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
txt = read_file(spec)
for line in txt.split('\n'):
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
name = module['module'][0]
try:
app_class = get_class(easyblock, name=name)
app = app_class(spec, debug=options.debug, robot_path=options.robot)
_log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError, err:
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent)
def cleanup_logfile_and_exit(logfile, testing, doexit):
"""Cleanup the logfile and exit"""
if not testing and logfile is not None:
os.remove(logfile)
print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing)
if doexit:
sys.exit(0)
def cleanup(logfile, tempdir, testing):
"""Cleanup the specified log file and the tmp directory, if desired."""
if build_option('cleanup_tmpdir') and not testing:
if logfile is not None:
try:
for log in [logfile] + glob.glob('%s.[0-9]*' % logfile):
os.remove(log)
except OSError, err:
raise EasyBuildError("Failed to remove log file(s) %s*: %s",
logfile, err)
print_msg("Tmporary log file(s) %s* have been removed." %
(logfile),
log=None,
silent=testing)
if tempdir is not None:
try:
shutil.rmtree(tempdir, ignore_errors=True)
except OSError, err:
raise EasyBuildError(
"Failed to remove temporary directory %s: %s", tempdir,
err)
print_msg("Temporary directory %s has been removed." % tempdir,
log=None,
silent=testing)
def obtain_path(specs,
paths,
try_to_generate=False,
exit_on_error=True,
silent=False):
"""Obtain a path for an easyconfig that matches the given specifications."""
# if no easyconfig files/paths were provided, but we did get a software name,
# we can try and find a suitable easyconfig ourselves, or generate one if we can
(generated, fn) = obtain_ec_for(specs, paths, None)
if not generated:
return (fn, generated)
else:
# if an easyconfig was generated, make sure we're allowed to use it
if try_to_generate:
print_msg(
"Generated an easyconfig file %s, going to use it now..." % fn,
silent=silent)
return (fn, generated)
else:
try:
os.remove(fn)
except OSError, err:
print_warning(
"Failed to remove generated easyconfig file %s: %s" %
(fn, err))
print_error((
"Unable to find an easyconfig for the given specifications: %s; "
"to make EasyBuild try to generate a matching easyconfig, "
"use the --try-X options ") % specs,
log=_log,
exit_on_error=exit_on_error)
def complete(self):
"""
Complete a bulk job submission.
Release all user holds on submitted jobs, and disconnect from server.
"""
for job in self._submitted:
if job.has_holds():
self.log.info("releasing user hold on job %s" % job.jobid)
job.release_hold()
self.disconnect_from_server()
# print list of submitted jobs
submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted])
print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log)
# determine leaf nodes in dependency graph, and report them
all_deps = set()
for job in self._submitted:
all_deps = all_deps.union(job.deps)
leaf_nodes = []
for job in self._submitted:
if job.jobid not in all_deps:
leaf_nodes.append(str(job.jobid).split('.')[0])
self.log.info("Job ids of leaf nodes in dep. graph: %s" % ','.join(leaf_nodes))
def build(module, options, log, origEnviron, exitOnFailure=True):
"""
Build the software
"""
spec = module["spec"]
print_msg("processing EasyBuild easyconfig %s" % spec, log)
## Restore original environment
log.info("Resetting environment")
filetools.errorsFoundInLog = 0
if not filetools.modifyEnv(os.environ, origEnviron):
error("Failed changing the environment back to original")
cwd = os.getcwd()
## Load easyblock
easyblock = options.easyblock
if not easyblock:
## Try to look in .eb file
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
for line in open(spec).readlines():
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
name = module["module"][0]
try:
app_class = get_class(easyblock, log, name=name)
app = app_class(spec, debug=options.debug)
log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError, err:
error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg))
def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
"""
Run hook with specified label.
:param label: name of hook
:param hooks: list of defined hooks
:param pre_step_hook: indicates whether hook to run is a pre-step hook
:param post_step_hook: indicates whether hook to run is a post-step hook
:param args: arguments to pass to hook function
:param msg: custom message that is printed when hook is called
"""
hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
if hook:
if args is None:
args = []
if pre_step_hook:
label = 'pre-' + label
elif post_step_hook:
label = 'post-' + label
if msg is None:
msg = "Running %s hook..." % label
print_msg(msg)
_log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
hook(*args)
def check_tool(tool_name, min_tool_version=None):
"""
This function is a predicate check for the existence of tool_name on the system PATH.
If min_tool_version is not None, it will check that the version has an equal or higher value.
"""
tool_path = which(tool_name)
if not tool_path:
return False
print_msg("{0} tool found at {1}".format(tool_name, tool_path))
if not min_tool_version:
return True
version_cmd = "{0} --version".format(tool_name)
out, ec = run_cmd(version_cmd,
simple=False,
trace=False,
force_in_dry_run=True)
if ec:
raise EasyBuildError(
"Error running '{0}' for tool {1} with output: {2}".format(
version_cmd, tool_name, out))
res = re.search("\d+\.\d+(\.\d+)?", out.strip())
if not res:
raise EasyBuildError(
"Error parsing version for tool {0}".format(tool_name))
tool_version = res.group(0)
version_ok = LooseVersion(
str(min_tool_version)) <= LooseVersion(tool_version)
if version_ok:
print_msg("{0} version '{1}' is {2} or higher ... OK".format(
tool_name, tool_version, min_tool_version))
return version_ok
def generate_recipe(self):
"""
This method will make use of resolve_template and resolve_template_data methods
in order to generate the container recipe.
"""
template = self.resolve_template()
data = self.resolve_template_data()
if self.img_name:
file_label = os.path.splitext(self.img_name)[0]
else:
file_label = data['mod_names'].split(' ')[0].replace('/', '-')
recipe_path = os.path.join(
self.container_path, "%s.%s" % (self.RECIPE_FILE_NAME, file_label))
if os.path.exists(recipe_path):
if build_option('force'):
print_msg(
"WARNING: overwriting existing container recipe at %s due to --force"
% recipe_path)
else:
raise EasyBuildError(
"Container recipe at %s already exists, not overwriting it without --force",
recipe_path)
recipe_content = template % data
write_file(recipe_path, recipe_content)
print_msg("%s definition file created at %s" %
(self.RECIPE_FILE_NAME, recipe_path),
log=self.log)
return recipe_path
def configure_step(self):
"""Custom configuration procedure for TAU: template configuration options before using them."""
if self.cc is None or self.cxx is None or self.fortran is None:
raise EasyBuildError("Compiler family not supported yet: %s", self.toolchain.comp_family())
if self.mpi_inc_dir is None or self.mpi_lib_dir is None:
raise EasyBuildError("Failed to determine MPI include/library paths, no MPI available in toolchain?")
# make sure selected default TAU makefile will be available
avail_makefiles = ['Makefile.' + l for l in self.variant_labels]
if self.cfg['tau_makefile'] not in avail_makefiles:
raise EasyBuildError("Specified tau_makefile %s will not be available (only: %s)",
self.cfg['tau_makefile'], avail_makefiles)
# inform which backend/variant is being handled
backend = (['tau'] + self.cfg['extra_backends'])[self.variant_index // 3]
variant = ['mpi', 'openmp', 'hybrid'][self.variant_index % 3]
print_msg("starting with %s backend (%s variant)" % (backend, variant), log=self.log, silent=self.silent)
self.cfg['configopts'] = self.cfg['configopts'] % {
'backend_opt': self.backend_opts[backend],
'cc': self.cc,
'cxx': self.cxx,
'fortran': self.fortran,
'mpi_inc_dir': self.mpi_inc_dir,
'mpi_lib_dir': self.mpi_lib_dir,
'opt_pkgs_opts': self.opt_pkgs_opts,
}
for key in ['preconfigopts', 'configopts', 'prebuildopts', 'preinstallopts']:
self.log.debug("%s for TAU (variant index: %s): %s", key, self.variant_index, self.cfg[key])
super(EB_TAU, self).configure_step()
self.variant_index += 1
def generate_recipe(self):
"""
This method will make use of resolve_template and resolve_template_data methods
in order to generate the container recipe.
"""
template = self.resolve_template()
data = self.resolve_template_data()
if self.img_name:
file_label = os.path.splitext(self.img_name)[0]
else:
file_label = data['mod_names'].split(' ')[0].replace('/', '-')
recipe_path = os.path.join(self.container_path, "%s.%s" % (self.RECIPE_FILE_NAME, file_label))
if os.path.exists(recipe_path):
if build_option('force'):
print_msg("WARNING: overwriting existing container recipe at %s due to --force" % recipe_path)
else:
raise EasyBuildError("Container recipe at %s already exists, not overwriting it without --force",
recipe_path)
recipe_content = template % data
write_file(recipe_path, recipe_content)
print_msg("%s definition file created at %s" % (self.RECIPE_FILE_NAME, recipe_path), log=self.log)
return recipe_path
def run_contrib_checks(ecs):
"""Run contribution check on specified easyconfigs."""
def print_result(checks_passed, label):
"""Helper function to print result of last group of checks."""
if checks_passed:
print_msg("\n>> All %s checks PASSed!" % label, prefix=False)
else:
print_msg("\n>> One or more %s checks FAILED!" % label,
prefix=False)
# start by running style checks
style_check_ok = cmdline_easyconfigs_style_check(ecs)
print_result(style_check_ok, "style")
# check whether SHA256 checksums are in place
print_msg("\nChecking for SHA256 checksums in %d easyconfig(s)...\n" %
len(ecs),
prefix=False)
sha256_checksums_ok = True
for ec in ecs:
sha256_checksum_fails = check_sha256_checksums([ec])
if sha256_checksum_fails:
sha256_checksums_ok = False
msgs = ['[FAIL] %s' % ec.path] + sha256_checksum_fails
else:
msgs = ['[PASS] %s' % ec.path]
print_msg('\n'.join(msgs), prefix=False)
print_result(sha256_checksums_ok, "SHA256 checksums")
return style_check_ok and sha256_checksums_ok
def find_easyconfigs_by_specs(build_specs,
robot_path,
try_to_generate,
testing=False):
"""Find easyconfigs by build specifications."""
generated, ec_file = obtain_ec_for(build_specs, robot_path, None)
if generated:
if try_to_generate:
print_msg(
"Generated an easyconfig file %s, going to use it now..." %
ec_file,
silent=testing)
else:
# (try to) cleanup
try:
os.remove(ec_file)
except OSError as err:
_log.warning(
"Failed to remove generated easyconfig file %s: %s" %
(ec_file, err))
# don't use a generated easyconfig unless generation was requested (using a --try-X option)
raise EasyBuildError(
"Unable to find an easyconfig for the given specifications: %s; "
"to make EasyBuild try to generate a matching easyconfig, "
"use the --try-X options ", build_specs)
return [(ec_file, generated)]
def print_result(checks_passed, label):
"""Helper function to print result of last group of checks."""
if checks_passed:
print_msg("\n>> All %s checks PASSed!" % label, prefix=False)
else:
print_msg("\n>> One or more %s checks FAILED!" % label,
prefix=False)
def cleanup(logfile, tempdir, testing, silent=False):
"""
Cleanup the specified log file and the tmp directory, if desired.
@param logfile: path to log file to clean up
@param tempdir: path to temporary directory to clean up
@param testing: are we in testing mode? if so, don't actually clean up anything
@param silent: be silent (don't print anything to stdout)
"""
if build_option('cleanup_tmpdir') and not testing:
if logfile is not None:
try:
for log in [logfile] + glob.glob('%s.[0-9]*' % logfile):
os.remove(log)
except OSError, err:
raise EasyBuildError("Failed to remove log file(s) %s*: %s", logfile, err)
print_msg("Temporary log file(s) %s* have been removed." % (logfile), log=None, silent=testing or silent)
if tempdir is not None:
try:
shutil.rmtree(tempdir, ignore_errors=True)
except OSError, err:
raise EasyBuildError("Failed to remove temporary directory %s: %s", tempdir, err)
print_msg("Temporary directory %s has been removed." % tempdir, log=None, silent=testing or silent)
def cmdline_easyconfigs_style_check(ecs):
"""
Run easyconfigs style check of each of the specified easyconfigs, triggered from 'eb' command line
:param ecs: list of easyconfigs to check, could be either file paths or EasyConfig instances
:return: True when style check passed on all easyconfig files, False otherwise
"""
print_msg("\nRunning style check on %d easyconfig(s)...\n" % len(ecs), prefix=False)
style_check_passed = True
for ec in ecs:
# if an EasyConfig instance is provided, just grab the corresponding file path
if isinstance(ec, EasyConfig):
path = ec.path
elif isinstance(ec, basestring):
path = ec
else:
raise EasyBuildError("Value of unknown type encountered in cmdline_easyconfigs_style_check: %s (type: %s)",
ec, type(ec))
if check_easyconfigs_style([path]) == 0:
res = 'PASS'
else:
res = 'FAIL'
style_check_passed = False
print_msg('[%s] %s' % (res, path), prefix=False)
return style_check_passed
def configure_step(self):
"""Custom configuration procedure for Paraver: template configuration options before using them."""
component = self.components[self.current_component]
try:
os.chdir(component)
except OSError as err:
raise EasyBuildError("Failed to move to %s: %s" % (component, err))
self.log.info("Customized start directory for component %s: %s",
component, os.getcwd())
print_msg("starting with component %s" % component, log=self.log)
# check for required Boost dependency
boost = get_software_root('Boost')
if not boost:
raise EasyBuildError("Boost is not available as a dependency")
if not self.wxpropgrid:
raise EasyBuildError(
"wxPropertyGrid is not available as a dependency")
self.cfg['configopts'] = self.cfg['configopts'] % {
'boost': boost,
'installdir': self.installdir,
'wxpropgrid': self.wxpropgrid,
}
super(EB_Paraver, self).configure_step()
def run_contrib_checks(ecs):
"""Run contribution check on specified easyconfigs."""
def print_result(checks_passed, label):
"""Helper function to print result of last group of checks."""
if checks_passed:
print_msg("\n>> All %s checks PASSed!" % label, prefix=False)
else:
print_msg("\n>> One or more %s checks FAILED!" % label, prefix=False)
# start by running style checks
style_check_ok = cmdline_easyconfigs_style_check(ecs)
print_result(style_check_ok, "style")
# check whether SHA256 checksums are in place
print_msg("\nChecking for SHA256 checksums in %d easyconfig(s)...\n" % len(ecs), prefix=False)
sha256_checksums_ok = True
for ec in ecs:
sha256_checksum_fails = check_sha256_checksums([ec])
if sha256_checksum_fails:
sha256_checksums_ok = False
msgs = ['[FAIL] %s' % ec.path] + sha256_checksum_fails
else:
msgs = ['[PASS] %s' % ec.path]
print_msg('\n'.join(msgs), prefix=False)
print_result(sha256_checksums_ok, "SHA256 checksums")
return style_check_ok and sha256_checksums_ok
def build_and_install_software(module, options, origEnviron, exitOnFailure=True, silent=False):
"""
Build the software
"""
spec = module['spec']
print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
# restore original environment
_log.info("Resetting environment")
filetools.errorsFoundInLog = 0
modify_env(os.environ, origEnviron)
cwd = os.getcwd()
# load easyblock
easyblock = options.easyblock
if not easyblock:
# try to look in .eb file
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
txt = read_file(spec)
for line in txt.split('\n'):
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
name = module['module'][0]
try:
app_class = get_class(easyblock, name=name)
app = app_class(spec, debug=options.debug, robot_path=options.robot)
_log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError, err:
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent)
def init_repo(path, repo_name, silent=False):
"""
Initialize a new Git repository at the specified location.
:param path: location where Git repository should be initialized
:param repo_name: name of Git repository
:param silent: keep quiet (don't print any messages)
"""
repo_path = os.path.join(path, repo_name)
# copy or init git working directory
git_working_dirs_path = build_option('git_working_dirs_path')
if git_working_dirs_path:
workdir = os.path.join(git_working_dirs_path, repo_name)
if os.path.exists(workdir):
try:
print_msg("copying %s..." % workdir, silent=silent)
shutil.copytree(workdir, repo_path)
except OSError as err:
raise EasyBuildError(
"Failed to copy git working dir %s to %s: %s", workdir,
repo_path, err)
if not os.path.exists(repo_path):
mkdir(repo_path, parents=True)
try:
repo = git.Repo.init(repo_path)
except GitCommandError as err:
raise EasyBuildError("Failed to init git repo at %s: %s", repo_path,
err)
_log.debug("temporary git working directory ready at %s", repo_path)
return repo
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts))
else:
raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
restore_env(orig_env)
def complete(self):
"""
Complete a bulk job submission.
Create engine, and progress it until all jobs have terminated.
"""
# create an instance of `Engine` using the list of configuration files
try:
self._engine = create_engine(*self.config_files,
resource_errors_are_fatal=True)
except gc3libs.exceptions.Error as err:
raise EasyBuildError("Failed to create GC3Pie engine: %s", err)
# make sure that all job log files end up in the same directory, rather than renaming the output directory
# see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine
self._engine.retrieve_overwrites = True
# `Engine.stats()` (which is used later on in `_print_status_report()`)
# changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible
# with both
try:
self._engine.init_stats_for(Application)
except AttributeError:
_log.debug("No `init_stats_for` method in the Engine class;"
" assuming pre-2.5.0 GC3Pie and ignoring error.")
# Add your application to the engine. This will NOT submit
# your application yet, but will make the engine *aware* of
# the application.
self._engine.add(self.jobs)
# in case you want to select a specific resource, call
target_resource = build_option('job_target_resource')
if target_resource:
res = self._engine.select_resource(target_resource)
if res == 0:
raise EasyBuildError(
"Failed to select target resource '%s' in GC3Pie",
target_resource)
# Periodically check the status of your application.
while self.jobs.execution.state != Run.State.TERMINATED:
# `Engine.progress()` will do the GC3Pie magic:
# submit new jobs, update status of submitted jobs, get
# results of terminating jobs etc...
self._engine.progress()
# report progress
self._print_status_report()
# Wait a few seconds...
time.sleep(self.poll_interval)
# final status report
print_msg("Done processing jobs",
log=self.log,
silent=build_option('silent'))
self._print_status_report()
def dump_env_script(easyconfigs):
"""
Dump source scripts that set up build environment for specified easyconfigs.
:param easyconfigs: list of easyconfigs to generate scripts for
"""
ecs_and_script_paths = []
for easyconfig in easyconfigs:
script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0]
ecs_and_script_paths.append((easyconfig['ec'], script_path))
# don't just overwrite existing scripts
existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)]
if existing_scripts:
if build_option('force'):
_log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts))
else:
raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s",
' '.join(existing_scripts))
orig_env = copy.deepcopy(os.environ)
for ec, script_path in ecs_and_script_paths:
# obtain EasyBlock instance
app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
app = app_class(ec)
# mimic dry run, and keep quiet
app.dry_run = app.silent = app.toolchain.dry_run = True
# prepare build environment (in dry run mode)
app.check_readiness_step()
app.prepare_step(start_dir=False)
# compose script
ecfile = os.path.basename(ec.path)
script_lines = [
"#!/bin/bash",
"# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
"# usage: source %s" % os.path.basename(script_path),
]
script_lines.extend(['', "# toolchain & dependency modules"])
if app.toolchain.modules:
script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
else:
script_lines.append("# (no modules loaded)")
script_lines.extend(['', "# build environment"])
if app.toolchain.vars:
env_vars = sorted(app.toolchain.vars.items())
script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
else:
script_lines.append("# (no build environment defined)")
write_file(script_path, '\n'.join(script_lines))
print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
restore_env(orig_env)
def error(message, exitCode=1, optparser=None):
"""
Print error message and exit EasyBuild
"""
print_msg("ERROR: %s\n" % message)
if optparser:
optparser.print_help()
sys.exit(exitCode)
def search_file(paths, query, short=False, ignore_dirs=None, silent=False):
"""
Search for a particular file (only prints)
"""
if ignore_dirs is None:
ignore_dirs = ['.git', '.svn']
if not isinstance(ignore_dirs, list):
raise EasyBuildError(
"search_file: ignore_dirs (%s) should be of type list, not %s",
ignore_dirs, type(ignore_dirs))
# compile regex, case-insensitive
query = re.compile(query, re.I)
var_lines = []
hit_lines = []
var_index = 1
var = None
for path in paths:
hits = []
hit_in_path = False
print_msg("Searching (case-insensitive) for '%s' in %s " %
(query.pattern, path),
log=_log,
silent=silent)
for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
for filename in filenames:
if query.search(filename):
if not hit_in_path:
var = "CFGS%d" % var_index
var_index += 1
hit_in_path = True
hits.append(os.path.join(dirpath, filename))
# do not consider (certain) hidden directories
# note: we still need to consider e.g., .local !
# replace list elements using [:], so os.walk doesn't process deleted directories
# see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
dirnames[:] = [d for d in dirnames if not d in ignore_dirs]
hits = sorted(hits)
if hits:
common_prefix = det_common_path_prefix(hits)
if short and common_prefix is not None and len(
common_prefix) > len(var) * 2:
var_lines.append("%s=%s" % (var, common_prefix))
hit_lines.extend([
" * %s" %
os.path.join('$%s' % var, fn[len(common_prefix) + 1:])
for fn in hits
])
else:
hit_lines.extend([" * %s" % fn for fn in hits])
for line in var_lines + hit_lines:
print_msg(line, log=_log, silent=silent, prefix=False)
def install_step(self):
"""Install components, if specified."""
comp_cnt = len(self.cfg['components'])
for idx, cfg in enumerate(self.comp_cfgs):
easyblock = cfg.get('easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError(
"No easyblock specified for component %s v%s", cfg['name'],
cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError(
"The '%s' easyblock can not be used to install components in a bundle",
easyblock)
print_msg("installing bundle component %s v%s (%d/%d)..." %
(cfg['name'], cfg['version'], idx + 1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s",
cfg['name'], cfg['version'], easyblock)
comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg)
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
# figure out correct start directory
comp.guess_start_dir()
# need to run fetch_patches to ensure per-component patches are applied
comp.fetch_patches()
# location of first unpacked source is used to determine where to apply patch(es)
comp.src = [{'finalpath': comp.cfg['start_dir']}]
# run relevant steps
for step_name in ['patch', 'configure', 'build', 'install']:
if step_name in cfg['skipsteps']:
comp.log.info("Skipping '%s' step for component %s v%s",
step_name, cfg['name'], cfg['version'])
else:
comp.run_step(
step_name,
[lambda x: getattr(x, '%s_step' % step_name)])
# update environment to ensure stuff provided by former components can be picked up by latter components
# once the installation is finalised, this is handled by the generated module
reqs = comp.make_module_req_guess()
for envvar in reqs:
curr_val = os.getenv(envvar, '')
curr_paths = curr_val.split(os.pathsep)
for subdir in reqs[envvar]:
path = os.path.join(self.installdir, subdir)
if path not in curr_paths:
if curr_val:
new_val = '%s:%s' % (path, curr_val)
else:
new_val = path
env.setvar(envvar, new_val)
def configure_step(self):
"""Add some extra configure options."""
# also build shared libraries (not enabled by default)
self.cfg.update('configopts', "--enable-shared")
if self.toolchain.options['pic']:
# Enforce consistency.
self.cfg.update('configopts', "--with-pic")
if LooseVersion(self.version) >= LooseVersion('2.0') and LooseVersion(self.version) < LooseVersion('2.1'):
# the code in libint is automatically generated and hence it is in some
# parts so complex that -O2 or -O3 compiler optimization takes forever
self.cfg.update('configopts', "--with-cxx-optflags='-O1'")
elif LooseVersion(self.version) >= LooseVersion('2.1'):
# pass down $CXXFLAGS to --with-cxxgen-optflags configure option;
# mainly to avoid warning about it not being set (but $CXXFLAGS is picked up anyway in practice)
self.cfg.update('configopts', "--with-cxxgen-optflags='%s'" % os.getenv('CXXFLAGS'))
if LooseVersion(self.version) >= LooseVersion('2.6.0'):
# Libint 2.6.0 requires first compiling the Libint compiler,
# by running configure with appropriate options, followed by 'make export'
# and unpacking the resulting source tarball;
# see https://github.com/evaleev/libint/wiki#compiling-libint-compiler
# CMake is recommended, but configuring with Fortran support doesn't work correctly yet in Libint 2.6.0
# so stick to traditional configure script for now
print_msg("configuring Libint compiler...")
# first run autogen.sh script to generate initial configure script
run_cmd("./autogen.sh")
cmd = ' '.join([
self.cfg['preconfigopts'],
'./configure',
self.cfg['configopts'],
self.cfg['libint_compiler_configopts'],
])
run_cmd(cmd)
print_msg("generating Libint library...")
run_cmd("make export")
source_fn = 'libint-%s.tgz' % self.version
if os.path.exists(source_fn):
extract_file(source_fn, os.getcwd(), change_into_dir=False)
change_dir('libint-%s' % self.version)
else:
raise EasyBuildError("Could not find generated source tarball after 'make export'!")
# --enable-fortran is only a known configure option for Libint library, not for Libint compiler,
# so only add --enable-fortran *after* configuring & generating Libint compiler
if self.cfg['with_fortran']:
self.cfg.update('configopts', '--enable-fortran')
super(EB_Libint, self).configure_step()
def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False):
"""
Search for a particular file (only prints)
"""
if ignore_dirs is None:
ignore_dirs = ['.git', '.svn']
if not isinstance(ignore_dirs, list):
raise EasyBuildError("search_file: ignore_dirs (%s) should be of type list, not %s",
ignore_dirs, type(ignore_dirs))
# compile regex, case-insensitive
query = re.compile(query, re.I)
var_lines = []
hit_lines = []
var_index = 1
var = None
for path in paths:
hits = []
hit_in_path = False
if not terse:
print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
for filename in filenames:
if query.search(filename):
if not hit_in_path:
var = "CFGS%d" % var_index
var_index += 1
hit_in_path = True
if filename_only:
hits.append(filename)
else:
hits.append(os.path.join(dirpath, filename))
# do not consider (certain) hidden directories
# note: we still need to consider e.g., .local !
# replace list elements using [:], so os.walk doesn't process deleted directories
# see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
dirnames[:] = [d for d in dirnames if d not in ignore_dirs]
hits = sorted(hits)
if hits and not terse:
common_prefix = det_common_path_prefix(hits)
if short and common_prefix is not None and len(common_prefix) > len(var) * 2:
var_lines.append("%s=%s" % (var, common_prefix))
hit_lines.extend([" * %s" % os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in hits])
else:
hit_lines.extend([" * %s" % fn for fn in hits])
if terse:
for line in hits:
print(line)
else:
for line in var_lines + hit_lines:
print_msg(line, log=_log, silent=silent, prefix=False)
def cleanup(logfile, tempdir, testing):
"""Cleanup the specified log file and the tmp directory"""
if not testing and logfile is not None:
os.remove(logfile)
print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing)
if not testing and tempdir is not None:
shutil.rmtree(tempdir, ignore_errors=True)
print_msg('temporary directory %s has been removed.' % (tempdir), log=None, silent=testing)
def complete(self):
"""
Complete a bulk job submission.
Create engine, and progress it until all jobs have terminated.
"""
# create an instance of `Engine` using the list of configuration files
try:
self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True)
except gc3libs.exceptions.Error as err:
raise EasyBuildError("Failed to create GC3Pie engine: %s", err)
# make sure that all job log files end up in the same directory, rather than renaming the output directory
# see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine
self._engine.retrieve_overwrites = True
# some sites may not be happy with flooding the cluster with build jobs...
self._engine.max_in_flight = build_option('job_max_jobs')
# `Engine.stats()` (which is used later on in `_print_status_report()`)
# changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible
# with both
try:
self._engine.init_stats_for(Application)
except AttributeError:
_log.debug("No `init_stats_for` method in the Engine class;"
" assuming pre-2.5.0 GC3Pie and ignoring error.")
# Add your application to the engine. This will NOT submit
# your application yet, but will make the engine *aware* of
# the application.
self._engine.add(self.jobs)
# in case you want to select a specific resource, call
target_resource = build_option('job_target_resource')
if target_resource:
res = self._engine.select_resource(target_resource)
if res == 0:
raise EasyBuildError("Failed to select target resource '%s' in GC3Pie", target_resource)
# Periodically check the status of your application.
while self.jobs.execution.state != Run.State.TERMINATED:
# `Engine.progress()` will do the GC3Pie magic:
# submit new jobs, update status of submitted jobs, get
# results of terminating jobs etc...
self._engine.progress()
# report progress
self._print_status_report()
# Wait a few seconds...
time.sleep(self.poll_interval)
# final status report
print_msg("Done processing jobs", log=self.log, silent=build_option('silent'))
self._print_status_report()
def cleanup(logfile, tempdir, testing):
"""Cleanup the specified log file and the tmp directory"""
if not testing and logfile is not None:
try:
for log in glob.glob('%s*' % logfile):
os.remove(log)
except OSError, err:
raise EasyBuildError("Failed to remove log file(s) %s*: %s", logfile, err)
print_msg('temporary log file(s) %s* have been removed.' % (logfile), log=None, silent=testing)
def guess_start_dir(self):
"""Guess start directory and inform which backend/variant is being handled next."""
super(EB_TAU, self).guess_start_dir()
# inform which backend/variant is being handled next
backend = (['tau'] + self.cfg['extra_backends'])[self.variant_index //
3]
variant = ['mpi', 'openmp', 'hybrid'][self.variant_index % 3]
print_msg("starting with %s backend (%s variant)" % (backend, variant),
log=self.log)
def print_dry_run(easyconfigs, short=False, build_specs=None):
"""
Print dry run information
@param easyconfigs: list of easyconfig files
@param short: print short output (use a variable for the common prefix)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
lines = []
if build_option('robot_path') is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
lines.append(
"Dry run: printing build status of easyconfigs and dependencies")
all_specs = resolve_dependencies(easyconfigs,
build_specs=build_specs,
retain_all_deps=True)
unbuilt_specs = skip_available(all_specs, testing=True)
dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front)
listed_ec_paths = [spec['spec'] for spec in easyconfigs]
var_name = 'CFGS'
common_prefix = det_common_path_prefix(
[spec['spec'] for spec in all_specs])
# only allow short if common prefix is long enough
short = short and common_prefix is not None and len(
common_prefix) > len(var_name) * 2
for spec in all_specs:
if spec in unbuilt_specs:
ans = ' '
elif build_option('force') and spec['spec'] in listed_ec_paths:
ans = 'F'
else:
ans = 'x'
if spec['ec'].short_mod_name != spec['ec'].full_mod_name:
mod = "%s | %s" % (spec['ec'].mod_subdir,
spec['ec'].short_mod_name)
else:
mod = spec['ec'].full_mod_name
if short:
item = os.path.join('$%s' % var_name,
spec['spec'][len(common_prefix) + 1:])
else:
item = spec['spec']
lines.append(dry_run_fmt % (ans, item, mod))
if short:
# insert after 'Dry run:' message
lines.insert(1, "%s=%s" % (var_name, common_prefix))
silent = build_option('silent')
print_msg('\n'.join(lines), log=_log, silent=silent, prefix=False)
def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=False):
"""
Set up repository by checking out specified branch from repository at specified URL.
@param git_repo: git.Repo instance
@param github_url: URL to GitHub repository
@param target_account: name of GitHub account that owns GitHub repository at specified URL
@param branch_name: name of branch to check out
@param silent: keep quiet (don't print any messages)
"""
_log.debug("Cloning from %s", github_url)
# salt to use for names of remotes/branches that are created
salt = ''.join(random.choice(string.letters) for _ in range(5))
remote_name = 'pr_target_account_%s_%s' % (target_account, salt)
origin = git_repo.create_remote(remote_name, github_url)
if not origin.exists():
raise EasyBuildError("%s does not exist?", github_url)
# git fetch
# can't use --depth to only fetch a shallow copy, since pushing to another repo from a shallow copy doesn't work
print_msg("fetching branch '%s' from %s..." % (branch_name, github_url), silent=silent)
try:
res = origin.fetch()
except GitCommandError as err:
raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err)
if res:
if res[0].flags & res[0].ERROR:
raise EasyBuildError("Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note)
else:
_log.debug("Fetched branch '%s' from remote %s (note: %s)", branch_name, origin, res[0].note)
else:
raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin)
# git checkout -b <branch>; git pull
if hasattr(origin.refs, branch_name):
origin_branch = getattr(origin.refs, branch_name)
else:
raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url)
_log.debug("Checking out branch '%s' from remote %s", branch_name, github_url)
try:
origin_branch.checkout(b=branch_name)
except GitCommandError as err:
alt_branch = '%s_%s' % (branch_name, salt)
_log.debug("Trying to work around checkout error ('%s') by using different branch name '%s'", err, alt_branch)
try:
origin_branch.checkout(b=alt_branch, force=True)
except GitCommandError as err:
raise EasyBuildError("Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err)
return remote_name
def run_check(msg, expected_stdout='', expected_stderr='', **kwargs):
"""Helper function to check stdout/stderr produced via print_msg"""
self.mock_stdout(True)
self.mock_stderr(True)
print_msg(msg, **kwargs)
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stdout, expected_stdout)
self.assertEqual(stderr, expected_stderr)
def run_check(msg, args, expected_stdout='', expected_stderr='', **kwargs):
"""Helper function to check stdout/stderr produced via print_msg."""
self.mock_stdout(True)
self.mock_stderr(True)
print_msg(msg, *args, **kwargs)
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stdout, expected_stdout)
self.assertEqual(stderr, expected_stderr)
def install_step(self):
"""
Install step: install each 'source file' one by one.
Installing the Intel compilers could be done via a single installation file (HPC Toolkit),
or with separate installation files (patch releases of the C++ and Fortran compilers).
"""
srcs = self.src[:]
cnt = len(srcs)
for idx, src in enumerate(srcs):
print_msg("installing part %d/%s (%s)..." % (idx + 1, cnt, src['name']))
self.src = [src]
super(EB_intel_minus_compilers, self).install_step()
def _print_status_report(self):
"""
Print a job status report to STDOUT and the log file.
The number of jobs in each state is reported; the
figures are extracted from the `stats()` method of the
currently-running GC3Pie engine.
"""
stats = self._engine.stats(only=Application)
states = ', '.join(["%d %s" % (stats[s], s.lower()) for s in stats if s != 'total' and stats[s]])
print_msg("GC3Pie job overview: %s (total: %s)" % (states, self.job_cnt),
log=self.log, silent=build_option('silent'))
def cleanup(logfile, tempdir, testing):
"""Cleanup the specified log file and the tmp directory"""
if not testing and logfile is not None:
try:
for log in glob.glob('%s*' % logfile):
os.remove(log)
except OSError, err:
raise EasyBuildError("Failed to remove log file(s) %s*: %s",
logfile, err)
print_msg('temporary log file(s) %s* have been removed.' % (logfile),
log=None,
silent=testing)
def build_image(self, dockerfile):
ec = self.easyconfigs[-1]['ec']
module_name = self.mns.det_full_module_name(ec)
tempdir = tempfile.mkdtemp(prefix='easybuild-docker')
container_name = self.img_name or "%s:latest" % module_name.replace('/', '-')
docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.'])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd)
run_cmd(docker_cmd, path=tempdir, stream_output=True)
print_msg("Docker image created at %s" % container_name, log=self.log)
rmtree2(tempdir)
def skip_available(easyconfigs, testing=False):
"""Skip building easyconfigs for which a module is already available."""
avail_modules = modules_tool().available()
easyconfigs, check_easyconfigs = [], easyconfigs
for ec in check_easyconfigs:
module = ec['full_mod_name']
if module in avail_modules:
msg = "%s is already installed (module found), skipping" % module
print_msg(msg, log=_log, silent=testing)
_log.info(msg)
else:
_log.debug("%s is not installed yet, so retaining it" % module)
easyconfigs.append(ec)
return easyconfigs
def search_file(paths, query, build_options=None, short=False):
"""
Search for a particular file (only prints)
"""
if build_options is None:
build_options = {}
ignore_dirs = build_options.get('ignore_dirs', ['.git', '.svn'])
if not isinstance(ignore_dirs, list):
_log.error("search_file: ignore_dirs (%s) should be of type list, not %s" % (ignore_dirs, type(ignore_dirs)))
silent = build_options.get('silent', False)
var_lines = []
hit_lines = []
var_index = 1
var = None
for path in paths:
hits = []
hit_in_path = False
print_msg("Searching (case-insensitive) for '%s' in %s " % (query, path), log=_log, silent=silent)
query = query.lower()
for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
for filename in filenames:
filename = os.path.join(dirpath, filename)
if filename.lower().find(query) != -1:
if not hit_in_path:
var = "CFGS%d" % var_index
var_index += 1
hit_in_path = True
hits.append(filename)
# do not consider (certain) hidden directories
# note: we still need to consider e.g., .local !
# replace list elements using [:], so os.walk doesn't process deleted directories
# see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
dirnames[:] = [d for d in dirnames if not d in ignore_dirs]
if hits:
common_prefix = det_common_path_prefix(hits)
if short and common_prefix is not None and len(common_prefix) > len(var) * 2:
var_lines.append("%s=%s" % (var, common_prefix))
hit_lines.extend([" * %s" % os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in hits])
else:
hit_lines.extend([" * %s" % fn for fn in hits])
for line in var_lines + hit_lines:
print_msg(line, log=_log, silent=silent, prefix=False)
def complete(self):
"""
Complete a bulk job submission.
Release all user holds on submitted jobs, and disconnect from server.
"""
job_ids = []
for job in self._submitted:
if job.job_specs['hold']:
self.log.info("releasing user hold on job %s" % job.jobid)
job_ids.append(job.jobid)
run_cmd("scontrol release %s" % ' '.join(job_ids), trace=False)
submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted])
print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log)
def install_step(self):
"""Install components, if specified."""
comp_cnt = len(self.cfg['components'])
for idx, cfg in enumerate(self.comp_cfgs):
easyblock = cfg.get('easyblock') or self.cfg['default_easyblock']
if easyblock is None:
raise EasyBuildError("No easyblock specified for component %s v%s", cfg['name'], cfg['version'])
elif easyblock == 'Bundle':
raise EasyBuildError("The '%s' easyblock can not be used to install components in a bundle", easyblock)
print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx+1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock)
comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg)
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
# figure out correct start directory
comp.guess_start_dir()
# need to run fetch_patches to ensure per-component patches are applied
comp.fetch_patches()
# location of first unpacked source is used to determine where to apply patch(es)
comp.src = [{'finalpath': comp.cfg['start_dir']}]
# run relevant steps
for step_name in ['patch', 'configure', 'build', 'install']:
if step_name in cfg['skipsteps']:
comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version'])
else:
comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
# update environment to ensure stuff provided by former components can be picked up by latter components
# once the installation is finalised, this is handled by the generated module
reqs = comp.make_module_req_guess()
for envvar in reqs:
curr_val = os.getenv(envvar, '')
curr_paths = curr_val.split(os.pathsep)
for subdir in reqs[envvar]:
path = os.path.join(self.installdir, subdir)
if path not in curr_paths:
if curr_val:
new_val = '%s:%s' % (path, curr_val)
else:
new_val = path
env.setvar(envvar, new_val)
def print_dry_run(easyconfigs, robot=None):
if robot is None:
print_msg("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
print_msg("Dry run: printing build status of easyconfigs and dependencies")
all_specs = resolve_dependencies(easyconfigs, robot, True)
unbuilt_specs = skip_available(all_specs, True)
dry_run_fmt = "%3s %s (module: %s)"
for spec in all_specs:
if spec in unbuilt_specs:
ans = '[ ]'
else:
ans = '[x]'
mod = det_full_module_name(spec['ec'])
print dry_run_fmt % (ans, spec['spec'], mod)
def skip_available(easyconfigs, testing=False):
"""Skip building easyconfigs for which a module is already available."""
m = modules_tool()
easyconfigs, check_easyconfigs = [], easyconfigs
for ec in check_easyconfigs:
module = ec["module"]
mod = "%s (version %s)" % (module[0], module[1])
modspath = mk_module_path(curr_module_paths() + [os.path.join(config.install_path("mod"), "all")])
if m.exists(module[0], module[1], modspath):
msg = "%s is already installed (module found in %s), skipping " % (mod, modspath)
print_msg(msg, log=_log, silent=testing)
_log.info(msg)
else:
_log.debug("%s is not installed yet, so retaining it" % mod)
easyconfigs.append(ec)
return easyconfigs