def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50, trace=True): """ Run specified interactive command (in a subshell) :param cmd: command to run :param qa: dictionary which maps question to answers :param no_qa: list of patters that are not questions :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param std_qa: dictionary which maps question regex patterns to answers :param path: path to execute the command is; current working directory is used if unspecified :param maxhits: maximum number of cycles (seconds) without being able to find a known question :param trace: print command being executed as part of trace output """ cwd = os.getcwd() if log_all or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err) _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None start_time = datetime.now() if trace: trace_txt = "running interactive command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd.strip()) # early exit in 'dry run' mode, after printing the command that would be run if build_option('extended_dry_run'): if path is None: path = cwd dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError, err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands=None): """ Custom sanity check for extensions, whether installed as stand-alone module or not """ if not self.cfg.get_ref('exts_filter'): self.cfg['exts_filter'] = exts_filter self.log.debug("starting sanity check for extension with filter %s", self.cfg.get_ref('exts_filter')) # for stand-alone installations that were done for multiple dependency versions (via multi_deps), # we need to perform the extension sanity check for each of them, by loading the corresponding modules first if self.cfg['multi_deps'] and not self.is_extension: multi_deps = self.cfg.get_parsed_multi_deps() lists_of_extra_modules = [[d['short_mod_name'] for d in deps] for deps in multi_deps] else: # make sure Extension sanity check step is run once, by using a single empty list of extra modules lists_of_extra_modules = [[]] for extra_modules in lists_of_extra_modules: fake_mod_data = None # only load fake module + extra modules for stand-alone installations (not for extensions), # since for extension the necessary modules should already be loaded at this point if not (self.is_extension or self.dry_run): # load fake module fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules) if extra_modules: info_msg = "Running extension sanity check with extra modules: %s" % ', '.join(extra_modules) self.log.info(info_msg) trace_msg(info_msg) # perform extension sanity check (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self) if fake_mod_data: # unload fake module and clean up self.clean_up_fake_module(fake_mod_data) if custom_paths or custom_commands or not self.is_extension: super(ExtensionEasyBlock, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands, extension=self.is_extension) # pass or fail sanity check if sanity_check_ok: self.log.info("Sanity check for %s successful!", self.name) else: if not self.is_extension: msg = "Sanity check for %s failed: %s" % (self.name, '; '.join(self.sanity_check_fail_msgs)) raise EasyBuildError(msg) return (sanity_check_ok, '; '.join(self.sanity_check_fail_msgs))
def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None, rpath_include_dirs=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment :param onlymod: boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). :param silent: keep quiet, or not (mostly relates to extended dry run output) :param loadmod: whether or not to (re)load the toolchain module, and the modules for the dependencies :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) :param rpath_include_dirs: extra directories to include in RPATH """ if loadmod: self._load_modules(silent=silent) if self.name != DUMMY_TOOLCHAIN_NAME: trace_msg("defining build environment for %s/%s toolchain" % (self.name, self.version)) if not self.dry_run: self._verify_toolchain() # Generate the variables to be set self.set_variables() # set the variables # onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug("prepare: do not set additional variables onlymod=%s", onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s", onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod, verbose=not silent) # consider f90cache first, since ccache can also wrap Fortran compilers for cache_tool in [F90CACHE, CCACHE]: if build_option('use_%s' % cache_tool): self.prepare_compiler_cache(cache_tool) if build_option('rpath'): if self.options.get('rpath', True): self.prepare_rpath_wrappers(rpath_filter_dirs, rpath_include_dirs) self.use_rpath = True else: self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option")
def _load_toolchain_module(self, silent=False): """Load toolchain module.""" tc_mod = self.det_short_module_name() if self.dry_run: dry_run_msg("Loading toolchain module...\n", silent=silent) # load toolchain module, or simulate load of toolchain components if it is not available if self.modules_tool.exist([tc_mod], skip_avail=True)[0]: self.modules_tool.load([tc_mod]) dry_run_msg("module load %s" % tc_mod, silent=silent) else: # first simulate loads for toolchain dependencies, if required information is available if self.tcdeps is not None: for tcdep in self.tcdeps: modname = tcdep['short_mod_name'] dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent) # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output) deproot = '$%s' % get_software_root_env_var_name( tcdep['name']) self._simulated_load_dependency_module( tcdep['name'], tcdep['version'], {'prefix': deproot}) dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent) # use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output) tcroot = '$%s' % get_software_root_env_var_name(self.name) self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot}) else: # make sure toolchain is available using short module name by running 'module use' on module path subdir if self.init_modpaths: mod_path_suffix = build_option('suffix_modules_path') for modpath in self.init_modpaths: self.modules_tool.prepend_module_path( os.path.join(install_path('mod'), mod_path_suffix, modpath)) # load modules for all dependencies self.log.debug("Loading module for toolchain: %s", tc_mod) trace_msg("loading toolchain module: " + tc_mod) self.modules_tool.load([tc_mod]) # append toolchain module to list of modules self.modules.append(tc_mod)
def _load_toolchain_module(self, silent=False): """Load toolchain module.""" tc_mod = self.det_short_module_name() if self.dry_run: dry_run_msg("Loading toolchain module...\n", silent=silent) # load toolchain module, or simulate load of toolchain components if it is not available if self.modules_tool.exist([tc_mod], skip_avail=True)[0]: self.modules_tool.load([tc_mod]) dry_run_msg("module load %s" % tc_mod, silent=silent) else: # first simulate loads for toolchain dependencies, if required information is available if self.tcdeps is not None: for tcdep in self.tcdeps: modname = tcdep['short_mod_name'] dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent) # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output) deproot = '$%s' % get_software_root_env_var_name(tcdep['name']) self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot}) dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent) # use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output) tcroot = '$%s' % get_software_root_env_var_name(self.name) self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot}) else: # make sure toolchain is available using short module name by running 'module use' on module path subdir if self.init_modpaths: mod_path_suffix = build_option('suffix_modules_path') for modpath in self.init_modpaths: self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) # load modules for all dependencies self.log.debug("Loading module for toolchain: %s", tc_mod) trace_msg("loading toolchain module: " + tc_mod) self.modules_tool.load([tc_mod]) # append toolchain module to list of modules self.modules.append(tc_mod)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50, trace=True): """ Run specified interactive command (in a subshell) :param cmd: command to run :param qa: dictionary which maps question to answers :param no_qa: list of patters that are not questions :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param std_qa: dictionary which maps question regex patterns to answers :param path: path to execute the command is; current working directory is used if unspecified :param maxhits: maximum number of cycles (seconds) without being able to find a known question :param trace: print command being executed as part of trace output """ cwd = os.getcwd() if log_all or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError( "Failed to open temporary log file for output of interactive command: %s", err) _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None start_time = datetime.now() if trace: trace_txt = "running interactive command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime( '%Y-%m-%d %H:%M:%S') trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd.strip()) # early exit in 'dry run' mode, after printing the command that would be run if build_option('extended_dry_run'): if path is None: path = cwd dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError as err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!" % cmd) # Part 1: process the QandA dictionary # given initial set of Q and A (in dict), return dict of reg. exp. and A # # make regular expression that matches the string with # - replace whitespace # - replace newline def escape_special(string): return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string) split = r'[\s\n]+' regSplit = re.compile(r"" + split) def process_QA(q, a_s): splitq = [escape_special(x) for x in regSplit.split(q)] regQtxt = split.join(splitq) + split.rstrip('+') + "*$" # add optional split at the end for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]: a_s[i] += '\n' regQ = re.compile(r"" + regQtxt) if regQ.search(q): return (a_s, regQ) else: raise EasyBuildError( "runqanda: Question %s converted in %s does not match itself", q, regQtxt) def check_answers_list(answers): """Make sure we have a list of answers (as strings).""" if isinstance(answers, string_type): answers = [answers] elif not isinstance(answers, list): raise EasyBuildError( "Invalid type for answer on %s, no string or list: %s (%s)", question, type(answers), answers) # list is manipulated when answering matching question, so return a copy return answers[:] new_qa = {} _log.debug("new_qa: ") for question, answers in qa.items(): answers = check_answers_list(answers) (answers, regQ) = process_QA(question, answers) new_qa[regQ] = answers _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ])) new_std_qa = {} if std_qa: for question, answers in std_qa.items(): regQ = re.compile(r"" + question + r"[\s\n]*$") answers = check_answers_list(answers) for i in [ idx for idx, a in enumerate(answers) if not a.endswith('\n') ]: answers[i] += '\n' new_std_qa[regQ] = answers _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ])) new_no_qa = [] if no_qa: # simple statements, can contain wildcards new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa] _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa]) # Part 2: Run the command and answer questions # - this needs asynchronous stdout # # Log command output if cmd_log: cmd_log.write("# output for interactive command: %s\n\n" % cmd) try: proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT, stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash') except OSError as err: raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err) ec = proc.poll() stdout_err = '' old_len_out = -1 hit_count = 0 while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working try: out = get_output_from_process(proc, asynchronous=True) if cmd_log: cmd_log.write(out) stdout_err += out # recv_some used by get_output_from_process for getting asynchronous output may throw exception except (IOError, Exception) as err: _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err) out = None hit = False for question, answers in new_qa.items(): res = question.search(stdout_err) if out and res: fa = answers[0] % res.groupdict() # cycle through list of answers last_answer = answers.pop(0) answers.append(last_answer) _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:]) asyncprocess.send_all(proc, fa) hit = True break if not hit: for question, answers in new_std_qa.items(): res = question.search(stdout_err) if out and res: fa = answers[0] % res.groupdict() # cycle through list of answers last_answer = answers.pop(0) answers.append(last_answer) _log.debug( "List of answers for question %s after cycling: %s", question.pattern, answers) _log.debug("run_cmd_qa answer %s std question %s out %s", fa, question.pattern, stdout_err[-50:]) asyncprocess.send_all(proc, fa) hit = True break if not hit: if len(stdout_err) > old_len_out: old_len_out = len(stdout_err) else: noqa = False for r in new_no_qa: if r.search(stdout_err): _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:]) noqa = True if not noqa: hit_count += 1 else: hit_count = 0 else: hit_count = 0 if hit_count > maxhits: # explicitly kill the child process before exiting try: os.killpg(proc.pid, signal.SIGKILL) os.kill(proc.pid, signal.SIGKILL) except OSError as err: _log.debug( "run_cmd_qa exception caught when killing child process: %s", err) _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err) raise EasyBuildError( "run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s", cmd, maxhits, stdout_err[-500:]) # the sleep below is required to avoid exiting on unknown 'questions' too early (see above) time.sleep(1) ec = proc.poll() # Process stopped. Read all remaining data try: if proc.stdout: out = get_output_from_process(proc) stdout_err += out if cmd_log: cmd_log.write(out) cmd_log.close() except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(cwd) except OSError as err: raise EasyBuildError( "Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None): """ Run specified command (in a subshell) :param cmd: command to run :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param inp: the input given to the command via stdin :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param log_output: indicate whether all output of command should be logged to a separate temporary logfile :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output :param shell: allow commands to not run in a shell (especially useful for cmd lists) :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout """ cwd = os.getcwd() if isinstance(cmd, string_type): cmd_msg = cmd.strip() elif isinstance(cmd, list): cmd_msg = ' '.join(cmd) else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError( "Failed to open temporary log file for output of command: %s", err) _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info( "Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg) stream_output = True if stream_output: print_msg("(streaming) output for command '%s':" % cmd_msg) start_time = datetime.now() if trace: trace_txt = "running command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime( '%Y-%m-%d %H:%M:%S') trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) if inp: trace_txt += "\t[input: %s]\n" % inp trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd_msg) # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) if not force_in_dry_run and build_option('extended_dry_run'): if path is None: path = cwd if verbose: dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) # make sure we get the type of the return value right if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError as err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!", cmd) if cmd_log: cmd_log.write("# output for command: %s\n\n" % cmd_msg) exec_cmd = "/bin/bash" if not shell: if isinstance(cmd, list): exec_cmd = None cmd.insert(0, '/usr/bin/env') elif isinstance(cmd, string_type): cmd = '/usr/bin/env %s' % cmd else: raise EasyBuildError( "Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) _log.info('running cmd: %s ' % cmd) try: proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) if inp: proc.stdin.write(inp.encode()) proc.stdin.close() # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead if stream_output: read_size = 128 else: read_size = 1024 * 8 ec = proc.poll() stdouterr = '' while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working output = get_output_from_process(proc, read_size=read_size) if cmd_log: cmd_log.write(output) if stream_output: sys.stdout.write(output) stdouterr += output ec = proc.poll() # read remaining data (all of it) output = get_output_from_process(proc) if cmd_log: cmd_log.write(output) cmd_log.close() if stream_output: sys.stdout.write(output) stdouterr += output if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(cwd) except OSError as err: raise EasyBuildError( "Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, regexp=True, stream_output=None, trace=True, output=''): """ Complete running of command represented by passed subprocess.Popen instance. :param proc: subprocess.Popen instance representing running command :param cmd: command being run :param owd: original working directory :param start_time: start time of command (datetime instance) :param cmd_log: log file to print command output to :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param stream_output: enable streaming command output to stdout :param trace: print command being executed as part of trace output """ # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead if stream_output: read_size = 128 else: read_size = 1024 * 8 stdouterr = output ec = proc.poll() while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working output = get_output_from_process(proc, read_size=read_size) if cmd_log: cmd_log.write(output) if stream_output: sys.stdout.write(output) stdouterr += output ec = proc.poll() # read remaining data (all of it) output = get_output_from_process(proc) proc.stdout.close() if cmd_log: cmd_log.write(output) cmd_log.close() if stream_output: sys.stdout.write(output) stdouterr += output if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(owd) except OSError as err: raise EasyBuildError( "Failed to return to %s after executing command: %s", owd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False): """ Run specified command (in a subshell) :param cmd: command to run :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param inp: the input given to the command via stdin :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param log_output: indicate whether all output of command should be logged to a separate temporary logfile :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) """ cwd = os.getcwd() if isinstance(cmd, string_type): cmd_msg = cmd.strip() elif isinstance(cmd, list): cmd_msg = ' '.join(cmd) else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) if shell is None: shell = True if isinstance(cmd, list): raise EasyBuildError( "When passing cmd as a list then `shell` must be set explictely! " "Note that all elements of the list but the first are treated as arguments " "to the shell and NOT to the command to be executed!") if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError( "Failed to open temporary log file for output of command: %s", err) _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info( "Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg) stream_output = True if stream_output: print_msg("(streaming) output for command '%s':" % cmd_msg) start_time = datetime.now() if trace: trace_txt = "running command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime( '%Y-%m-%d %H:%M:%S') trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) if inp: trace_txt += "\t[input: %s]\n" % inp trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd_msg) # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) if not force_in_dry_run and build_option('extended_dry_run'): if path is None: path = cwd if verbose: dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) # make sure we get the type of the return value right if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError as err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!", cmd) if cmd_log: cmd_log.write("# output for command: %s\n\n" % cmd_msg) exec_cmd = "/bin/bash" if not shell: if isinstance(cmd, list): exec_cmd = None cmd.insert(0, '/usr/bin/env') elif isinstance(cmd, string_type): cmd = '/usr/bin/env %s' % cmd else: raise EasyBuildError( "Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) _log.info('running cmd: %s ' % cmd) try: proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) if inp: proc.stdin.write(inp.encode()) proc.stdin.close() if asynchronous: return (proc, cmd, cwd, start_time, cmd_log) else: return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, regexp=regexp, stream_output=stream_output, trace=trace)
def _load_dependencies_modules(self, silent=False): """Load modules for dependencies, and handle special cases like external modules.""" dep_mods = [dep['short_mod_name'] for dep in self.dependencies] if self.dry_run: dry_run_msg("\nLoading modules for dependencies...\n", silent=silent) mods_exist = self.modules_tool.exist(dep_mods) # load available modules for dependencies, simulate load for others for dep, dep_mod_exists in zip(self.dependencies, mods_exist): mod_name = dep['short_mod_name'] if dep_mod_exists: self.modules_tool.load([mod_name]) dry_run_msg("module load %s" % mod_name, silent=silent) else: dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent) # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output) if not dep['external_module']: deproot = '$%s' % get_software_root_env_var_name( dep['name']) self._simulated_load_dependency_module( dep['name'], dep['version'], {'prefix': deproot}) else: # load modules for all dependencies self.log.debug("Loading modules for dependencies: %s", dep_mods) self.modules_tool.load(dep_mods) if self.dependencies: build_dep_mods = [ dep['short_mod_name'] for dep in self.dependencies if dep['build_only'] ] if build_dep_mods: trace_msg("loading modules for build dependencies:") for dep_mod in build_dep_mods: trace_msg(' * ' + dep_mod) else: trace_msg("(no build dependencies specified)") run_dep_mods = [ dep['short_mod_name'] for dep in self.dependencies if not dep['build_only'] ] if run_dep_mods: trace_msg("loading modules for (runtime) dependencies:") for dep_mod in run_dep_mods: trace_msg(' * ' + dep_mod) else: trace_msg("(no (runtime) dependencies specified)") # append dependency modules to list of modules self.modules.extend(dep_mods) # define $EBROOT* and $EBVERSION* for external modules, if metadata is available for dep in [d for d in self.dependencies if d['external_module']]: mod_name = dep['full_mod_name'] metadata = dep['external_module_metadata'] self.log.debug("Metadata for external module %s: %s", mod_name, metadata) names = metadata.get('name', []) versions = metadata.get('version', [None] * len(names)) self.log.debug( "Defining $EB* environment variables for external module %s using names %s, versions %s", mod_name, names, versions) for name, version in zip(names, versions): self._simulated_load_dependency_module(name, version, metadata, verbose=True)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None): """ Run specified command (in a subshell) :param cmd: command to run :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param inp: the input given to the command via stdin :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param log_output: indicate whether all output of command should be logged to a separate temporary logfile :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output :param shell: allow commands to not run in a shell (especially useful for cmd lists) :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout """ cwd = os.getcwd() if isinstance(cmd, basestring): cmd_msg = cmd.strip() elif isinstance(cmd, list): cmd_msg = ' '.join(cmd) else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError("Failed to open temporary log file for output of command: %s", err) _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg) stream_output = True if stream_output: print_msg("(streaming) output for command '%s':" % cmd_msg) start_time = datetime.now() if trace: trace_txt = "running command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd_msg) # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) if not force_in_dry_run and build_option('extended_dry_run'): if path is None: path = cwd if verbose: dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) # make sure we get the type of the return value right if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError as err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!", cmd) if cmd_log: cmd_log.write("# output for command: %s\n\n" % cmd_msg) exec_cmd = "/bin/bash" if not shell: if isinstance(cmd, list): exec_cmd = None cmd.insert(0, '/usr/bin/env') elif isinstance(cmd, basestring): cmd = '/usr/bin/env %s' % cmd else: raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) _log.info('running cmd: %s ' % cmd) try: proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) if inp: proc.stdin.write(inp) proc.stdin.close() # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead if stream_output: read_size = 128 else: read_size = 1024 * 8 ec = proc.poll() stdouterr = '' while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working output = proc.stdout.read(read_size) if cmd_log: cmd_log.write(output) if stream_output: sys.stdout.write(output) stdouterr += output ec = proc.poll() # read remaining data (all of it) output = proc.stdout.read() if cmd_log: cmd_log.write(output) cmd_log.close() if stream_output: sys.stdout.write(output) stdouterr += output if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(cwd) except OSError as err: raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50, trace=True): """ Run specified interactive command (in a subshell) :param cmd: command to run :param qa: dictionary which maps question to answers :param no_qa: list of patters that are not questions :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param std_qa: dictionary which maps question regex patterns to answers :param path: path to execute the command is; current working directory is used if unspecified :param maxhits: maximum number of cycles (seconds) without being able to find a known question :param trace: print command being executed as part of trace output """ cwd = os.getcwd() if log_all or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err) _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None start_time = datetime.now() if trace: trace_txt = "running interactive command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd.strip()) # early exit in 'dry run' mode, after printing the command that would be run if build_option('extended_dry_run'): if path is None: path = cwd dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError as err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!" % cmd) # Part 1: process the QandA dictionary # given initial set of Q and A (in dict), return dict of reg. exp. and A # # make regular expression that matches the string with # - replace whitespace # - replace newline def escape_special(string): return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string) split = '[\s\n]+' regSplit = re.compile(r"" + split) def process_QA(q, a_s): splitq = [escape_special(x) for x in regSplit.split(q)] regQtxt = split.join(splitq) + split.rstrip('+') + "*$" # add optional split at the end for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]: a_s[i] += '\n' regQ = re.compile(r"" + regQtxt) if regQ.search(q): return (a_s, regQ) else: raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt) def check_answers_list(answers): """Make sure we have a list of answers (as strings).""" if isinstance(answers, basestring): answers = [answers] elif not isinstance(answers, list): raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)", question, type(answers), answers) # list is manipulated when answering matching question, so return a copy return answers[:] new_qa = {} _log.debug("new_qa: ") for question, answers in qa.items(): answers = check_answers_list(answers) (answers, regQ) = process_QA(question, answers) new_qa[regQ] = answers _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ])) new_std_qa = {} if std_qa: for question, answers in std_qa.items(): regQ = re.compile(r"" + question + r"[\s\n]*$") answers = check_answers_list(answers) for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]: answers[i] += '\n' new_std_qa[regQ] = answers _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ])) new_no_qa = [] if no_qa: # simple statements, can contain wildcards new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa] _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa]) # Part 2: Run the command and answer questions # - this needs asynchronous stdout # # Log command output if cmd_log: cmd_log.write("# output for interactive command: %s\n\n" % cmd) try: p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, stdin=PIPE, close_fds=True, executable="/bin/bash") except OSError as err: raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err) ec = p.poll() stdout_err = '' old_len_out = -1 hit_count = 0 while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working try: out = recv_some(p) if cmd_log: cmd_log.write(out) stdout_err += out # recv_some may throw Exception except (IOError, Exception) as err: _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err) out = None hit = False for question, answers in new_qa.items(): res = question.search(stdout_err) if out and res: fa = answers[0] % res.groupdict() # cycle through list of answers last_answer = answers.pop(0) answers.append(last_answer) _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:]) send_all(p, fa) hit = True break if not hit: for question, answers in new_std_qa.items(): res = question.search(stdout_err) if out and res: fa = answers[0] % res.groupdict() # cycle through list of answers last_answer = answers.pop(0) answers.append(last_answer) _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers) _log.debug("run_cmd_qa answer %s std question %s out %s", fa, question.pattern, stdout_err[-50:]) send_all(p, fa) hit = True break if not hit: if len(stdout_err) > old_len_out: old_len_out = len(stdout_err) else: noqa = False for r in new_no_qa: if r.search(stdout_err): _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:]) noqa = True if not noqa: hit_count += 1 else: hit_count = 0 else: hit_count = 0 if hit_count > maxhits: # explicitly kill the child process before exiting try: os.killpg(p.pid, signal.SIGKILL) os.kill(p.pid, signal.SIGKILL) except OSError as err: _log.debug("run_cmd_qa exception caught when killing child process: %s", err) _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err) raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s", cmd, maxhits, stdout_err[-500:]) # the sleep below is required to avoid exiting on unknown 'questions' too early (see above) time.sleep(1) ec = p.poll() # Process stopped. Read all remaining data try: if p.stdout: out = p.stdout.read() stdout_err += out if cmd_log: cmd_log.write(out) cmd_log.close() except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(cwd) except OSError as err: raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None): """ Run specified command (in a subshell) :param cmd: command to run :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param inp: the input given to the command via stdin :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param log_output: indicate whether all output of command should be logged to a separate temporary logfile :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output :param shell: allow commands to not run in a shell (especially useful for cmd lists) :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout """ cwd = os.getcwd() if isinstance(cmd, basestring): cmd_msg = cmd.strip() elif isinstance(cmd, list): cmd_msg = ' '.join(cmd) else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError("Failed to open temporary log file for output of command: %s", err) _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg) stream_output = True if stream_output: print_msg("(streaming) output for command '%s':" % cmd_msg) start_time = datetime.now() if trace: trace_txt = "running command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd_msg) # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) if not force_in_dry_run and build_option('extended_dry_run'): if path is None: path = cwd if verbose: dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) # make sure we get the type of the return value right if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError, err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!", cmd)
def _load_dependencies_modules(self, silent=False): """Load modules for dependencies, and handle special cases like external modules.""" dep_mods = [dep['short_mod_name'] for dep in self.dependencies] if self.dry_run: dry_run_msg("\nLoading modules for dependencies...\n", silent=silent) mods_exist = self.modules_tool.exist(dep_mods) # load available modules for dependencies, simulate load for others for dep, dep_mod_exists in zip(self.dependencies, mods_exist): mod_name = dep['short_mod_name'] if dep_mod_exists: self.modules_tool.load([mod_name]) dry_run_msg("module load %s" % mod_name, silent=silent) else: dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent) # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output) if not dep['external_module']: deproot = '$%s' % get_software_root_env_var_name(dep['name']) self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot}) else: # load modules for all dependencies self.log.debug("Loading modules for dependencies: %s", dep_mods) self.modules_tool.load(dep_mods) if self.dependencies: build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']] if build_dep_mods: trace_msg("loading modules for build dependencies:") for dep_mod in build_dep_mods: trace_msg(' * ' + dep_mod) else: trace_msg("(no build dependencies specified)") run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']] if run_dep_mods: trace_msg("loading modules for (runtime) dependencies:") for dep_mod in run_dep_mods: trace_msg(' * ' + dep_mod) else: trace_msg("(no (runtime) dependencies specified)") # append dependency modules to list of modules self.modules.extend(dep_mods) # define $EBROOT* and $EBVERSION* for external modules, if metadata is available for dep in [d for d in self.dependencies if d['external_module']]: mod_name = dep['full_mod_name'] metadata = dep['external_module_metadata'] self.log.debug("Metadata for external module %s: %s", mod_name, metadata) names = metadata.get('name', []) versions = metadata.get('version', [None] * len(names)) self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s", mod_name, names, versions) for name, version in zip(names, versions): self._simulated_load_dependency_module(name, version, metadata, verbose=True)
if stream_output: sys.stdout.write(output) stdouterr += output ec = proc.poll() # read remaining data (all of it) output = proc.stdout.read() if cmd_log: cmd_log.write(output) cmd_log.close() if stream_output: sys.stdout.write(output) stdouterr += output if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(cwd) except OSError, err: raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50, trace=True): """ Run specified interactive command (in a subshell) :param cmd: command to run :param qa: dictionary which maps question to answers
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, force_in_dry_run=False, verbose=True, shell=True, trace=True): """ Run specified command (in a subshell) :param cmd: command to run :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param inp: the input given to the command via stdin :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param log_output: indicate whether all output of command should be logged to a separate temporary logfile :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output :param shell: allow commands to not run in a shell (especially useful for cmd lists) :param trace: print command being executed as part of trace output """ cwd = os.getcwd() if isinstance(cmd, basestring): cmd_msg = cmd.strip() elif isinstance(cmd, list): cmd_msg = ' '.join(cmd) else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError( "Failed to open temporary log file for output of command: %s", err) _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None start_time = datetime.now() if trace: trace_txt = "running command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime( '%Y-%m-%d %H:%M:%S') trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd_msg) # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) if not force_in_dry_run and build_option('extended_dry_run'): if path is None: path = cwd if verbose: dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) # make sure we get the type of the return value right if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError, err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!", cmd)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50, trace=True): """ Run specified interactive command (in a subshell) :param cmd: command to run :param qa: dictionary which maps question to answers :param no_qa: list of patters that are not questions :param log_ok: only run output/exit code for failing commands (exit code non-zero) :param log_all: always log command output and exit code :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param std_qa: dictionary which maps question regex patterns to answers :param path: path to execute the command is; current working directory is used if unspecified :param maxhits: maximum number of cycles (seconds) without being able to find a known question :param trace: print command being executed as part of trace output """ cwd = os.getcwd() if log_all or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') os.close(fd) try: cmd_log = open(cmd_log_fn, 'w') except IOError as err: raise EasyBuildError( "Failed to open temporary log file for output of interactive command: %s", err) _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn)) else: cmd_log_fn, cmd_log = None, None start_time = datetime.now() if trace: trace_txt = "running interactive command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime( '%Y-%m-%d %H:%M:%S') trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd.strip()) # early exit in 'dry run' mode, after printing the command that would be run if build_option('extended_dry_run'): if path is None: path = cwd dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) dry_run_msg(" (in %s)" % path, silent=build_option('silent')) if simple: return True else: # output, exit code return ('', 0) try: if path: os.chdir(path) _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd())) except OSError, err: _log.warning("Failed to change to %s: %s" % (path, err)) _log.info("running cmd %s in non-existing directory, might fail!" % cmd)
# - otherwise the stdout/stderr buffer gets filled and it all stops working output = p.stdout.read(readSize) if cmd_log: cmd_log.write(output) stdouterr += output ec = p.poll() # read remaining data (all of it) output = p.stdout.read() if cmd_log: cmd_log.write(output) cmd_log.close() stdouterr += output if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: os.chdir(cwd) except OSError, err: raise EasyBuildError( "Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True,