def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
maxhits=50, trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands=None):
"""
Custom sanity check for extensions, whether installed as stand-alone module or not
"""
if not self.cfg.get_ref('exts_filter'):
self.cfg['exts_filter'] = exts_filter
self.log.debug("starting sanity check for extension with filter %s", self.cfg.get_ref('exts_filter'))
# for stand-alone installations that were done for multiple dependency versions (via multi_deps),
# we need to perform the extension sanity check for each of them, by loading the corresponding modules first
if self.cfg['multi_deps'] and not self.is_extension:
multi_deps = self.cfg.get_parsed_multi_deps()
lists_of_extra_modules = [[d['short_mod_name'] for d in deps] for deps in multi_deps]
else:
# make sure Extension sanity check step is run once, by using a single empty list of extra modules
lists_of_extra_modules = [[]]
for extra_modules in lists_of_extra_modules:
fake_mod_data = None
# only load fake module + extra modules for stand-alone installations (not for extensions),
# since for extension the necessary modules should already be loaded at this point
if not (self.is_extension or self.dry_run):
# load fake module
fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules)
if extra_modules:
info_msg = "Running extension sanity check with extra modules: %s" % ', '.join(extra_modules)
self.log.info(info_msg)
trace_msg(info_msg)
# perform extension sanity check
(sanity_check_ok, fail_msg) = Extension.sanity_check_step(self)
if fake_mod_data:
# unload fake module and clean up
self.clean_up_fake_module(fake_mod_data)
if custom_paths or custom_commands or not self.is_extension:
super(ExtensionEasyBlock, self).sanity_check_step(custom_paths=custom_paths,
custom_commands=custom_commands,
extension=self.is_extension)
# pass or fail sanity check
if sanity_check_ok:
self.log.info("Sanity check for %s successful!", self.name)
else:
if not self.is_extension:
msg = "Sanity check for %s failed: %s" % (self.name, '; '.join(self.sanity_check_fail_msgs))
raise EasyBuildError(msg)
return (sanity_check_ok, '; '.join(self.sanity_check_fail_msgs))
def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
- load modules for toolchain and dependencies
- generate extra variables and set them in the environment
:param onlymod: boolean/string to indicate if the toolchain should only load the environment
with module (True) or also set all other variables (False) like compiler CC etc
(If string: comma separated list of variables that will be ignored).
:param silent: keep quiet, or not (mostly relates to extended dry run output)
:param loadmod: whether or not to (re)load the toolchain module, and the modules for the dependencies
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
:param rpath_include_dirs: extra directories to include in RPATH
"""
if loadmod:
self._load_modules(silent=silent)
if self.name != DUMMY_TOOLCHAIN_NAME:
trace_msg("defining build environment for %s/%s toolchain" % (self.name, self.version))
if not self.dry_run:
self._verify_toolchain()
# Generate the variables to be set
self.set_variables()
# set the variables
# onlymod can be comma-separated string of variables not to be set
if onlymod == True:
self.log.debug("prepare: do not set additional variables onlymod=%s", onlymod)
self.generate_vars()
else:
self.log.debug("prepare: set additional variables onlymod=%s", onlymod)
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod, verbose=not silent)
# consider f90cache first, since ccache can also wrap Fortran compilers
for cache_tool in [F90CACHE, CCACHE]:
if build_option('use_%s' % cache_tool):
self.prepare_compiler_cache(cache_tool)
if build_option('rpath'):
if self.options.get('rpath', True):
self.prepare_rpath_wrappers(rpath_filter_dirs, rpath_include_dirs)
self.use_rpath = True
else:
self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option")
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname,
silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(
tcdep['name'])
self._simulated_load_dependency_module(
tcdep['name'], tcdep['version'],
{'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod,
silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version,
{'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(
os.path.join(install_path('mod'), mod_path_suffix,
modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def run_cmd_qa(cmd,
qa,
no_qa=None,
log_ok=True,
log_all=False,
simple=False,
regexp=True,
std_qa=None,
path=None,
maxhits=50,
trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of interactive command: %s",
err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" %
cmd)
# Part 1: process the QandA dictionary
# given initial set of Q and A (in dict), return dict of reg. exp. and A
#
# make regular expression that matches the string with
# - replace whitespace
# - replace newline
def escape_special(string):
return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
split = r'[\s\n]+'
regSplit = re.compile(r"" + split)
def process_QA(q, a_s):
splitq = [escape_special(x) for x in regSplit.split(q)]
regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
# add optional split at the end
for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
a_s[i] += '\n'
regQ = re.compile(r"" + regQtxt)
if regQ.search(q):
return (a_s, regQ)
else:
raise EasyBuildError(
"runqanda: Question %s converted in %s does not match itself",
q, regQtxt)
def check_answers_list(answers):
"""Make sure we have a list of answers (as strings)."""
if isinstance(answers, string_type):
answers = [answers]
elif not isinstance(answers, list):
raise EasyBuildError(
"Invalid type for answer on %s, no string or list: %s (%s)",
question, type(answers), answers)
# list is manipulated when answering matching question, so return a copy
return answers[:]
new_qa = {}
_log.debug("new_qa: ")
for question, answers in qa.items():
answers = check_answers_list(answers)
(answers, regQ) = process_QA(question, answers)
new_qa[regQ] = answers
_log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
new_std_qa = {}
if std_qa:
for question, answers in std_qa.items():
regQ = re.compile(r"" + question + r"[\s\n]*$")
answers = check_answers_list(answers)
for i in [
idx for idx, a in enumerate(answers)
if not a.endswith('\n')
]:
answers[i] += '\n'
new_std_qa[regQ] = answers
_log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
new_no_qa = []
if no_qa:
# simple statements, can contain wildcards
new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
_log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
# Part 2: Run the command and answer questions
# - this needs asynchronous stdout
# # Log command output
if cmd_log:
cmd_log.write("# output for interactive command: %s\n\n" % cmd)
try:
proc = asyncprocess.Popen(cmd,
shell=True,
stdout=asyncprocess.PIPE,
stderr=asyncprocess.STDOUT,
stdin=asyncprocess.PIPE,
close_fds=True,
executable='/bin/bash')
except OSError as err:
raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
ec = proc.poll()
stdout_err = ''
old_len_out = -1
hit_count = 0
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
try:
out = get_output_from_process(proc, asynchronous=True)
if cmd_log:
cmd_log.write(out)
stdout_err += out
# recv_some used by get_output_from_process for getting asynchronous output may throw exception
except (IOError, Exception) as err:
_log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
out = None
hit = False
for question, answers in new_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug("List of answers for question %s after cycling: %s",
question.pattern, answers)
_log.debug("run_cmd_qa answer %s question %s out %s", fa,
question.pattern, stdout_err[-50:])
asyncprocess.send_all(proc, fa)
hit = True
break
if not hit:
for question, answers in new_std_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug(
"List of answers for question %s after cycling: %s",
question.pattern, answers)
_log.debug("run_cmd_qa answer %s std question %s out %s",
fa, question.pattern, stdout_err[-50:])
asyncprocess.send_all(proc, fa)
hit = True
break
if not hit:
if len(stdout_err) > old_len_out:
old_len_out = len(stdout_err)
else:
noqa = False
for r in new_no_qa:
if r.search(stdout_err):
_log.debug("runqanda: noQandA found for out %s",
stdout_err[-50:])
noqa = True
if not noqa:
hit_count += 1
else:
hit_count = 0
else:
hit_count = 0
if hit_count > maxhits:
# explicitly kill the child process before exiting
try:
os.killpg(proc.pid, signal.SIGKILL)
os.kill(proc.pid, signal.SIGKILL)
except OSError as err:
_log.debug(
"run_cmd_qa exception caught when killing child process: %s",
err)
_log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
raise EasyBuildError(
"run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
cmd, maxhits, stdout_err[-500:])
# the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
time.sleep(1)
ec = proc.poll()
# Process stopped. Read all remaining data
try:
if proc.stdout:
out = get_output_from_process(proc)
stdout_err += out
if cmd_log:
cmd_log.write(out)
cmd_log.close()
except IOError as err:
_log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
if trace:
trace_msg("interactive command completed: exit %s, ran in %s" %
(ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError(
"Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok,
regexp)
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=True,
trace=True,
stream_output=None):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
"""
cwd = os.getcwd()
if isinstance(cmd, string_type):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of command: %s",
err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info(
"Auto-enabling streaming output of '%s' command because logging to stdout is enabled",
cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
if inp:
trace_txt += "\t[input: %s]\n" % inp
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
if cmd_log:
cmd_log.write("# output for command: %s\n\n" % cmd_msg)
exec_cmd = "/bin/bash"
if not shell:
if isinstance(cmd, list):
exec_cmd = None
cmd.insert(0, '/usr/bin/env')
elif isinstance(cmd, string_type):
cmd = '/usr/bin/env %s' % cmd
else:
raise EasyBuildError(
"Don't know how to prefix with /usr/bin/env for commands of type %s",
type(cmd))
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd,
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
executable=exec_cmd)
except OSError as err:
raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
if inp:
proc.stdin.write(inp.encode())
proc.stdin.close()
# use small read size when streaming output, to make it stream more fluently
# read size should not be too small though, to avoid too much overhead
if stream_output:
read_size = 128
else:
read_size = 1024 * 8
ec = proc.poll()
stdouterr = ''
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
output = get_output_from_process(proc, read_size=read_size)
if cmd_log:
cmd_log.write(output)
if stream_output:
sys.stdout.write(output)
stdouterr += output
ec = proc.poll()
# read remaining data (all of it)
output = get_output_from_process(proc)
if cmd_log:
cmd_log.write(output)
cmd_log.close()
if stream_output:
sys.stdout.write(output)
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" %
(ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError(
"Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok,
regexp)
def complete_cmd(proc,
cmd,
owd,
start_time,
cmd_log,
log_ok=True,
log_all=False,
simple=False,
regexp=True,
stream_output=None,
trace=True,
output=''):
"""
Complete running of command represented by passed subprocess.Popen instance.
:param proc: subprocess.Popen instance representing running command
:param cmd: command being run
:param owd: original working directory
:param start_time: start time of command (datetime instance)
:param cmd_log: log file to print command output to
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param stream_output: enable streaming command output to stdout
:param trace: print command being executed as part of trace output
"""
# use small read size when streaming output, to make it stream more fluently
# read size should not be too small though, to avoid too much overhead
if stream_output:
read_size = 128
else:
read_size = 1024 * 8
stdouterr = output
ec = proc.poll()
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
output = get_output_from_process(proc, read_size=read_size)
if cmd_log:
cmd_log.write(output)
if stream_output:
sys.stdout.write(output)
stdouterr += output
ec = proc.poll()
# read remaining data (all of it)
output = get_output_from_process(proc)
proc.stdout.close()
if cmd_log:
cmd_log.write(output)
cmd_log.close()
if stream_output:
sys.stdout.write(output)
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" %
(ec, time_str_since(start_time)))
try:
os.chdir(owd)
except OSError as err:
raise EasyBuildError(
"Failed to return to %s after executing command: %s", owd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok,
regexp)
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=None,
trace=True,
stream_output=None,
asynchronous=False):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
:param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
"""
cwd = os.getcwd()
if isinstance(cmd, string_type):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if shell is None:
shell = True
if isinstance(cmd, list):
raise EasyBuildError(
"When passing cmd as a list then `shell` must be set explictely! "
"Note that all elements of the list but the first are treated as arguments "
"to the shell and NOT to the command to be executed!")
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of command: %s",
err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info(
"Auto-enabling streaming output of '%s' command because logging to stdout is enabled",
cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
if inp:
trace_txt += "\t[input: %s]\n" % inp
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
if cmd_log:
cmd_log.write("# output for command: %s\n\n" % cmd_msg)
exec_cmd = "/bin/bash"
if not shell:
if isinstance(cmd, list):
exec_cmd = None
cmd.insert(0, '/usr/bin/env')
elif isinstance(cmd, string_type):
cmd = '/usr/bin/env %s' % cmd
else:
raise EasyBuildError(
"Don't know how to prefix with /usr/bin/env for commands of type %s",
type(cmd))
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd,
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
executable=exec_cmd)
except OSError as err:
raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
if inp:
proc.stdin.write(inp.encode())
proc.stdin.close()
if asynchronous:
return (proc, cmd, cwd, start_time, cmd_log)
else:
return complete_cmd(proc,
cmd,
cwd,
start_time,
cmd_log,
log_ok=log_ok,
log_all=log_all,
simple=simple,
regexp=regexp,
stream_output=stream_output,
trace=trace)
def _load_dependencies_modules(self, silent=False):
"""Load modules for dependencies, and handle special cases like external modules."""
dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
if self.dry_run:
dry_run_msg("\nLoading modules for dependencies...\n",
silent=silent)
mods_exist = self.modules_tool.exist(dep_mods)
# load available modules for dependencies, simulate load for others
for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
mod_name = dep['short_mod_name']
if dep_mod_exists:
self.modules_tool.load([mod_name])
dry_run_msg("module load %s" % mod_name, silent=silent)
else:
dry_run_msg("module load %s [SIMULATED]" % mod_name,
silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
if not dep['external_module']:
deproot = '$%s' % get_software_root_env_var_name(
dep['name'])
self._simulated_load_dependency_module(
dep['name'], dep['version'], {'prefix': deproot})
else:
# load modules for all dependencies
self.log.debug("Loading modules for dependencies: %s", dep_mods)
self.modules_tool.load(dep_mods)
if self.dependencies:
build_dep_mods = [
dep['short_mod_name'] for dep in self.dependencies
if dep['build_only']
]
if build_dep_mods:
trace_msg("loading modules for build dependencies:")
for dep_mod in build_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no build dependencies specified)")
run_dep_mods = [
dep['short_mod_name'] for dep in self.dependencies
if not dep['build_only']
]
if run_dep_mods:
trace_msg("loading modules for (runtime) dependencies:")
for dep_mod in run_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no (runtime) dependencies specified)")
# append dependency modules to list of modules
self.modules.extend(dep_mods)
# define $EBROOT* and $EBVERSION* for external modules, if metadata is available
for dep in [d for d in self.dependencies if d['external_module']]:
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Metadata for external module %s: %s", mod_name,
metadata)
names = metadata.get('name', [])
versions = metadata.get('version', [None] * len(names))
self.log.debug(
"Defining $EB* environment variables for external module %s using names %s, versions %s",
mod_name, names, versions)
for name, version in zip(names, versions):
self._simulated_load_dependency_module(name,
version,
metadata,
verbose=True)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
"""
cwd = os.getcwd()
if isinstance(cmd, basestring):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
if cmd_log:
cmd_log.write("# output for command: %s\n\n" % cmd_msg)
exec_cmd = "/bin/bash"
if not shell:
if isinstance(cmd, list):
exec_cmd = None
cmd.insert(0, '/usr/bin/env')
elif isinstance(cmd, basestring):
cmd = '/usr/bin/env %s' % cmd
else:
raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd)
except OSError as err:
raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
if inp:
proc.stdin.write(inp)
proc.stdin.close()
# use small read size when streaming output, to make it stream more fluently
# read size should not be too small though, to avoid too much overhead
if stream_output:
read_size = 128
else:
read_size = 1024 * 8
ec = proc.poll()
stdouterr = ''
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
output = proc.stdout.read(read_size)
if cmd_log:
cmd_log.write(output)
if stream_output:
sys.stdout.write(output)
stdouterr += output
ec = proc.poll()
# read remaining data (all of it)
output = proc.stdout.read()
if cmd_log:
cmd_log.write(output)
cmd_log.close()
if stream_output:
sys.stdout.write(output)
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
maxhits=50, trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
# Part 1: process the QandA dictionary
# given initial set of Q and A (in dict), return dict of reg. exp. and A
#
# make regular expression that matches the string with
# - replace whitespace
# - replace newline
def escape_special(string):
return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
split = '[\s\n]+'
regSplit = re.compile(r"" + split)
def process_QA(q, a_s):
splitq = [escape_special(x) for x in regSplit.split(q)]
regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
# add optional split at the end
for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
a_s[i] += '\n'
regQ = re.compile(r"" + regQtxt)
if regQ.search(q):
return (a_s, regQ)
else:
raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt)
def check_answers_list(answers):
"""Make sure we have a list of answers (as strings)."""
if isinstance(answers, basestring):
answers = [answers]
elif not isinstance(answers, list):
raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)",
question, type(answers), answers)
# list is manipulated when answering matching question, so return a copy
return answers[:]
new_qa = {}
_log.debug("new_qa: ")
for question, answers in qa.items():
answers = check_answers_list(answers)
(answers, regQ) = process_QA(question, answers)
new_qa[regQ] = answers
_log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
new_std_qa = {}
if std_qa:
for question, answers in std_qa.items():
regQ = re.compile(r"" + question + r"[\s\n]*$")
answers = check_answers_list(answers)
for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]:
answers[i] += '\n'
new_std_qa[regQ] = answers
_log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
new_no_qa = []
if no_qa:
# simple statements, can contain wildcards
new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
_log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
# Part 2: Run the command and answer questions
# - this needs asynchronous stdout
# # Log command output
if cmd_log:
cmd_log.write("# output for interactive command: %s\n\n" % cmd)
try:
p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, stdin=PIPE, close_fds=True, executable="/bin/bash")
except OSError as err:
raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
ec = p.poll()
stdout_err = ''
old_len_out = -1
hit_count = 0
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
try:
out = recv_some(p)
if cmd_log:
cmd_log.write(out)
stdout_err += out
# recv_some may throw Exception
except (IOError, Exception) as err:
_log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
out = None
hit = False
for question, answers in new_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
_log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:])
send_all(p, fa)
hit = True
break
if not hit:
for question, answers in new_std_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
_log.debug("run_cmd_qa answer %s std question %s out %s", fa, question.pattern, stdout_err[-50:])
send_all(p, fa)
hit = True
break
if not hit:
if len(stdout_err) > old_len_out:
old_len_out = len(stdout_err)
else:
noqa = False
for r in new_no_qa:
if r.search(stdout_err):
_log.debug("runqanda: noQandA found for out %s", stdout_err[-50:])
noqa = True
if not noqa:
hit_count += 1
else:
hit_count = 0
else:
hit_count = 0
if hit_count > maxhits:
# explicitly kill the child process before exiting
try:
os.killpg(p.pid, signal.SIGKILL)
os.kill(p.pid, signal.SIGKILL)
except OSError as err:
_log.debug("run_cmd_qa exception caught when killing child process: %s", err)
_log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
cmd, maxhits, stdout_err[-500:])
# the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
time.sleep(1)
ec = p.poll()
# Process stopped. Read all remaining data
try:
if p.stdout:
out = p.stdout.read()
stdout_err += out
if cmd_log:
cmd_log.write(out)
cmd_log.close()
except IOError as err:
_log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
if trace:
trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
"""
cwd = os.getcwd()
if isinstance(cmd, basestring):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
def _load_dependencies_modules(self, silent=False):
"""Load modules for dependencies, and handle special cases like external modules."""
dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
if self.dry_run:
dry_run_msg("\nLoading modules for dependencies...\n", silent=silent)
mods_exist = self.modules_tool.exist(dep_mods)
# load available modules for dependencies, simulate load for others
for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
mod_name = dep['short_mod_name']
if dep_mod_exists:
self.modules_tool.load([mod_name])
dry_run_msg("module load %s" % mod_name, silent=silent)
else:
dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
if not dep['external_module']:
deproot = '$%s' % get_software_root_env_var_name(dep['name'])
self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot})
else:
# load modules for all dependencies
self.log.debug("Loading modules for dependencies: %s", dep_mods)
self.modules_tool.load(dep_mods)
if self.dependencies:
build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']]
if build_dep_mods:
trace_msg("loading modules for build dependencies:")
for dep_mod in build_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no build dependencies specified)")
run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']]
if run_dep_mods:
trace_msg("loading modules for (runtime) dependencies:")
for dep_mod in run_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no (runtime) dependencies specified)")
# append dependency modules to list of modules
self.modules.extend(dep_mods)
# define $EBROOT* and $EBVERSION* for external modules, if metadata is available
for dep in [d for d in self.dependencies if d['external_module']]:
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
names = metadata.get('name', [])
versions = metadata.get('version', [None] * len(names))
self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s",
mod_name, names, versions)
for name, version in zip(names, versions):
self._simulated_load_dependency_module(name, version, metadata, verbose=True)
if stream_output:
sys.stdout.write(output)
stdouterr += output
ec = proc.poll()
# read remaining data (all of it)
output = proc.stdout.read()
if cmd_log:
cmd_log.write(output)
cmd_log.close()
if stream_output:
sys.stdout.write(output)
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError, err:
raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
maxhits=50, trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=True,
trace=True):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if isinstance(cmd, basestring):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of command: %s",
err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
def run_cmd_qa(cmd,
qa,
no_qa=None,
log_ok=True,
log_all=False,
simple=False,
regexp=True,
std_qa=None,
path=None,
maxhits=50,
trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of interactive command: %s",
err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" %
cmd)
# - otherwise the stdout/stderr buffer gets filled and it all stops working
output = p.stdout.read(readSize)
if cmd_log:
cmd_log.write(output)
stdouterr += output
ec = p.poll()
# read remaining data (all of it)
output = p.stdout.read()
if cmd_log:
cmd_log.write(output)
cmd_log.close()
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" %
(ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError, err:
raise EasyBuildError(
"Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok,
regexp)
def run_cmd_qa(cmd,
qa,
no_qa=None,
log_ok=True,
