def _setenv_variables(self, donotset=None, verbose=True):
"""Actually set the environment variables"""
self.log.debug("_setenv_variables: setting variables: donotset=%s" % donotset)
if self.dry_run:
dry_run_msg("Defining build environment...\n", silent=not verbose)
donotsetlist = []
if isinstance(donotset, str):
# TODO : more legacy code that should be using proper type
raise EasyBuildError("_setenv_variables: using commas-separated list. should be deprecated.")
elif isinstance(donotset, list):
donotsetlist = donotset
for key, val in sorted(self.vars.items()):
if key in donotsetlist:
self.log.debug("_setenv_variables: not setting environment variable %s (value: %s)." % (key, val))
continue
self.log.debug("_setenv_variables: setting environment variable %s to %s" % (key, val))
setvar(key, val, verbose=verbose)
# also set unique named variables that can be used in Makefiles
# - so you can have 'CFLAGS = $(EBVARCFLAGS)'
# -- 'CLFLAGS = $(CFLAGS)' gives '*** Recursive variable `CFLAGS'
# references itself (eventually). Stop' error
setvar("EBVAR%s" % key, val, verbose=False)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50):
"""
Run specified interactive command (in a subshell)
@param cmd: command to run
@param qa: dictionary which maps question to answers
@param no_qa: list of patters that are not questions
@param log_ok: only run output/exit code for failing commands (exit code non-zero)
@param log_all: always log command output and exit code
@param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
@param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
@param std_qa: dictionary which maps question regex patterns to answers
@param path: path to execute the command is; current working directory is used if unspecified
"""
cwd = os.getcwd()
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def apply_patch(patch_file, dest, fn=None, copy=False, level=None):
"""
Apply a patch to source code in directory dest
- assume unified diff created with "diff -ru old new"
"""
if build_option('extended_dry_run'):
# skip checking of files in dry run mode
patch_filename = os.path.basename(patch_file)
dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent'))
elif not os.path.isfile(patch_file):
raise EasyBuildError("Can't find patch %s: no such file", patch_file)
elif fn and not os.path.isfile(fn):
raise EasyBuildError("Can't patch file %s: no such file", fn)
elif not os.path.isdir(dest):
raise EasyBuildError("Can't patch directory %s: no such directory", dest)
# copy missing files
if copy:
if build_option('extended_dry_run'):
dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent'))
else:
try:
shutil.copy2(patch_file, dest)
_log.debug("Copied patch %s to dir %s" % (patch_file, dest))
# early exit, work is done after copying
return True
except IOError, err:
raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err)
def _load_modules(self, silent=False):
"""Load modules for toolchain and dependencies."""
if self.modules_tool is None:
raise EasyBuildError("No modules tool defined in Toolchain instance.")
if not self._toolchain_exists() and not self.dry_run:
raise EasyBuildError("No module found for toolchain: %s", self.mod_short_name)
if self.name == DUMMY_TOOLCHAIN_NAME:
if self.version == DUMMY_TOOLCHAIN_VERSION:
self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies')
if self.dry_run:
dry_run_msg("(no modules are loaded for a dummy-dummy toolchain)", silent=silent)
else:
self.log.info('prepare: toolchain dummy mode and loading dependencies')
self._load_dependencies_modules(silent=silent)
else:
# load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
self._load_toolchain_module(silent=silent)
self._load_dependencies_modules(silent=silent)
# include list of loaded modules in dry run output
if self.dry_run:
loaded_mods = self.modules_tool.list()
dry_run_msg("\nFull list of loaded modules:", silent=silent)
if loaded_mods:
for i, mod_name in enumerate([m['mod_name'] for m in loaded_mods]):
dry_run_msg(" %d) %s" % (i+1, mod_name), silent=silent)
else:
dry_run_msg(" (none)", silent=silent)
dry_run_msg('', silent=silent)
def _setenv_variables(self, donotset=None, verbose=True):
"""Actually set the environment variables"""
self.log.debug("_setenv_variables: setting variables: donotset=%s" %
donotset)
if self.dry_run:
dry_run_msg("Defining build environment...\n", silent=not verbose)
donotsetlist = []
if isinstance(donotset, str):
# TODO : more legacy code that should be using proper type
raise EasyBuildError(
"_setenv_variables: using commas-separated list. should be deprecated."
)
elif isinstance(donotset, list):
donotsetlist = donotset
for key, val in sorted(self.vars.items()):
if key in donotsetlist:
self.log.debug(
"_setenv_variables: not setting environment variable %s (value: %s)."
% (key, val))
continue
self.log.debug(
"_setenv_variables: setting environment variable %s to %s" %
(key, val))
setvar(key, val, verbose=verbose)
# also set unique named variables that can be used in Makefiles
# - so you can have 'CFLAGS = $(EBVARCFLAGS)'
# -- 'CLFLAGS = $(CFLAGS)' gives '*** Recursive variable `CFLAGS'
# references itself (eventually). Stop' error
setvar("EBVAR%s" % key, val, verbose=False)
def apply_regex_substitutions(path, regex_subs):
"""
Apply specified list of regex substitutions.
@param path: path to file to patch
@param regex_subs: list of substitutions to apply, specified as (<regexp pattern>, <replacement string>)
"""
# only report when in 'dry run' mode
if build_option('extended_dry_run'):
dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent'))
for regex, subtxt in regex_subs:
dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt))
else:
_log.debug("Applying following regex substitutions to %s: %s", path, regex_subs)
for i, (regex, subtxt) in enumerate(regex_subs):
regex_subs[i] = (re.compile(regex), subtxt)
try:
for line in fileinput.input(path, inplace=1, backup='.orig.eb'):
for regex, subtxt in regex_subs:
line = regex.sub(subtxt, line)
sys.stdout.write(line)
except OSError, err:
raise EasyBuildError("Failed to patch %s: %s", path, err)
def apply_patch(patch_file, dest, fn=None, copy=False, level=None):
"""
Apply a patch to source code in directory dest
- assume unified diff created with "diff -ru old new"
"""
if build_option('extended_dry_run'):
# skip checking of files in dry run mode
patch_filename = os.path.basename(patch_file)
dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent'))
elif not os.path.isfile(patch_file):
raise EasyBuildError("Can't find patch %s: no such file", patch_file)
elif fn and not os.path.isfile(fn):
raise EasyBuildError("Can't patch file %s: no such file", fn)
elif not os.path.isdir(dest):
raise EasyBuildError("Can't patch directory %s: no such directory", dest)
# copy missing files
if copy:
if build_option('extended_dry_run'):
dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent'))
else:
try:
shutil.copy2(patch_file, dest)
_log.debug("Copied patch %s to dir %s" % (patch_file, dest))
# early exit, work is done after copying
return True
except IOError, err:
raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err)
def _load_modules(self, silent=False):
"""Load modules for toolchain and dependencies."""
if self.modules_tool is None:
raise EasyBuildError("No modules tool defined in Toolchain instance.")
if not self._toolchain_exists() and not self.dry_run:
raise EasyBuildError("No module found for toolchain: %s", self.mod_short_name)
if self.name == DUMMY_TOOLCHAIN_NAME:
if self.version == DUMMY_TOOLCHAIN_VERSION:
self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies')
if self.dry_run:
dry_run_msg("(no modules are loaded for a dummy-dummy toolchain)", silent=silent)
else:
self.log.info('prepare: toolchain dummy mode and loading dependencies')
self._load_dependencies_modules(silent=silent)
else:
# load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
self._load_toolchain_module(silent=silent)
self._load_dependencies_modules(silent=silent)
# include list of loaded modules in dry run output
if self.dry_run:
loaded_mods = self.modules_tool.list()
dry_run_msg("\nFull list of loaded modules:", silent=silent)
if loaded_mods:
for i, mod_name in enumerate([m['mod_name'] for m in loaded_mods]):
dry_run_msg(" %d) %s" % (i+1, mod_name), silent=silent)
else:
dry_run_msg(" (none)", silent=silent)
dry_run_msg('', silent=silent)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
maxhits=50, trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def run_check(msg, args, expected_stdout='', **kwargs):
"""Helper function to check stdout/stderr produced via dry_run_msg."""
self.mock_stdout(True)
self.mock_stderr(True)
dry_run_msg(msg, *args, **kwargs)
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stdout, expected_stdout)
self.assertEqual(stderr, '')
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=True):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
"""
cwd = os.getcwd()
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" %
cmd)
def _load_modules(self, silent=False):
"""Load modules for toolchain and dependencies."""
if self.modules_tool is None:
raise EasyBuildError(
"No modules tool defined in Toolchain instance.")
if not self._toolchain_exists() and not self.dry_run:
raise EasyBuildError("No module found for toolchain: %s",
self.mod_short_name)
if self.is_system_toolchain():
self.log.info("Loading dependencies using system toolchain...")
self._load_dependencies_modules(silent=silent)
else:
# load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
self._load_toolchain_module(silent=silent)
self._load_dependencies_modules(silent=silent)
# include list of loaded modules in dry run output
if self.dry_run:
loaded_mods = self.modules_tool.list()
dry_run_msg("\nFull list of loaded modules:", silent=silent)
if loaded_mods:
for i, mod_name in enumerate(
[m['mod_name'] for m in loaded_mods]):
dry_run_msg(" %d) %s" % (i + 1, mod_name), silent=silent)
else:
dry_run_msg(" (none)", silent=silent)
dry_run_msg('', silent=silent)
def write_file(path, txt, append=False, forced=False):
"""Write given contents to file at given path (overwrites current file contents!)."""
# early exit in 'dry run' mode
if not forced and build_option('extended_dry_run'):
dry_run_msg("file written: %s" % path, silent=build_option('silent'))
return
# note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
try:
mkdir(os.path.dirname(path), parents=True)
with open(path, 'a' if append else 'w') as handle:
handle.write(txt)
except IOError, err:
raise EasyBuildError("Failed to write to %s: %s", path, err)
def write_file(path, txt, append=False, forced=False):
"""Write given contents to file at given path (overwrites current file contents!)."""
# early exit in 'dry run' mode
if not forced and build_option('extended_dry_run'):
dry_run_msg("file written: %s" % path, silent=build_option('silent'))
return
# note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
try:
mkdir(os.path.dirname(path), parents=True)
with open(path, 'a' if append else 'w') as handle:
handle.write(txt)
except IOError, err:
raise EasyBuildError("Failed to write to %s: %s", path, err)
def run_cmd_qa(cmd,
qa,
no_qa=None,
log_ok=True,
log_all=False,
simple=False,
regexp=True,
std_qa=None,
path=None,
maxhits=50):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
"""
cwd = os.getcwd()
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" %
cmd)
def unset_env_vars(keys, verbose=True):
"""
Unset the keys given in the environment
Returns a dict with the old values of the unset keys
"""
old_environ = {}
if keys and verbose and build_option('extended_dry_run'):
dry_run_msg("Undefining environment variables:\n", silent=build_option('silent'))
for key in keys:
if key in os.environ:
_log.info("Unsetting environment variable %s (value: %s)" % (key, os.environ[key]))
old_environ[key] = os.environ[key]
del os.environ[key]
if verbose and build_option('extended_dry_run'):
dry_run_msg(" unset %s # value was: %s" % (key, old_environ[key]), silent=build_option('silent'))
return old_environ
def unset_env_vars(keys, verbose=True):
"""
Unset the keys given in the environment
Returns a dict with the old values of the unset keys
"""
old_environ = {}
if keys and verbose and build_option('extended_dry_run'):
dry_run_msg("Undefining environment variables:\n", silent=build_option('silent'))
for key in keys:
if key in os.environ:
_log.info("Unsetting environment variable %s (value: %s)" % (key, os.environ[key]))
old_environ[key] = os.environ[key]
del os.environ[key]
if verbose and build_option('extended_dry_run'):
dry_run_msg(" unset %s # value was: %s" % (key, old_environ[key]), silent=build_option('silent'))
return old_environ
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
force_in_dry_run=False, verbose=True, shell=True):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
"""
cwd = os.getcwd()
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
def setvar(key, value, verbose=True):
"""
put key in the environment with value
tracks added keys until write_changes has been called
@param verbose: include message in dry run output for defining this environment variable
"""
if key in os.environ:
oldval_info = "previous value: '%s'" % os.environ[key]
else:
oldval_info = "previously undefined"
# os.putenv() is not necessary. os.environ will call this.
os.environ[key] = value
_changes[key] = value
_log.info("Environment variable %s set to %s (%s)", key, value, oldval_info)
if verbose and build_option('extended_dry_run'):
quoted_value = shell_quote(value)
if quoted_value[0] not in ['"', "'"]:
quoted_value = '"%s"' % quoted_value
dry_run_msg(" export %s=%s" % (key, quoted_value), silent=build_option('silent'))
def setvar(key, value, verbose=True):
"""
put key in the environment with value
tracks added keys until write_changes has been called
@param verbose: include message in dry run output for defining this environment variable
"""
if key in os.environ:
oldval_info = "previous value: '%s'" % os.environ[key]
else:
oldval_info = "previously undefined"
# os.putenv() is not necessary. os.environ will call this.
os.environ[key] = value
_changes[key] = value
_log.info("Environment variable %s set to %s (%s)", key, value, oldval_info)
if verbose and build_option('extended_dry_run'):
quoted_value = shell_quote(value)
if quoted_value[0] not in ['"', "'"]:
quoted_value = '"%s"' % quoted_value
dry_run_msg(" export %s=%s" % (key, quoted_value), silent=build_option('silent'))
def _load_dependencies_modules(self, silent=False):
"""Load modules for dependencies, and handle special cases like external modules."""
dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
if self.dry_run:
dry_run_msg("\nLoading modules for dependencies...\n", silent=silent)
mods_exist = self.modules_tool.exist(dep_mods)
# load available modules for dependencies, simulate load for others
for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
mod_name = dep['short_mod_name']
if dep_mod_exists:
self.modules_tool.load([mod_name])
dry_run_msg("module load %s" % mod_name, silent=silent)
else:
dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
if not dep['external_module']:
deproot = '$%s' % get_software_root_env_var_name(dep['name'])
self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot})
else:
# load modules for all dependencies
self.log.debug("Loading modules for dependencies: %s", dep_mods)
self.modules_tool.load(dep_mods)
if self.dependencies:
build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']]
if build_dep_mods:
trace_msg("loading modules for build dependencies:")
for dep_mod in build_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no build dependencies specified)")
run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']]
if run_dep_mods:
trace_msg("loading modules for (runtime) dependencies:")
for dep_mod in run_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no (runtime) dependencies specified)")
# append dependency modules to list of modules
self.modules.extend(dep_mods)
# define $EBROOT* and $EBVERSION* for external modules, if metadata is available
for dep in [d for d in self.dependencies if d['external_module']]:
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
names = metadata.get('name', [])
versions = metadata.get('version', [None] * len(names))
self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s",
mod_name, names, versions)
for name, version in zip(names, versions):
self._simulated_load_dependency_module(name, version, metadata, verbose=True)
def _load_dependencies_modules(self, silent=False):
"""Load modules for dependencies, and handle special cases like external modules."""
dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
if self.dry_run:
dry_run_msg("\nLoading modules for dependencies...\n", silent=silent)
mods_exist = self.modules_tool.exist(dep_mods)
# load available modules for dependencies, simulate load for others
for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
mod_name = dep['short_mod_name']
if dep_mod_exists:
self.modules_tool.load([mod_name])
dry_run_msg("module load %s" % mod_name, silent=silent)
else:
dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
if not dep['external_module']:
deproot = '$%s' % get_software_root_env_var_name(dep['name'])
self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot})
else:
# load modules for all dependencies
self.log.debug("Loading modules for dependencies: %s", dep_mods)
self.modules_tool.load(dep_mods)
if self.dependencies:
build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']]
if build_dep_mods:
trace_msg("loading modules for build dependencies:")
for dep_mod in build_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no build dependencies specified)")
run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']]
if run_dep_mods:
trace_msg("loading modules for (runtime) dependencies:")
for dep_mod in run_dep_mods:
trace_msg(' * ' + dep_mod)
else:
trace_msg("(no (runtime) dependencies specified)")
# append dependency modules to list of modules
self.modules.extend(dep_mods)
# define $EBROOT* and $EBVERSION* for external modules, if metadata is available
for dep in [d for d in self.dependencies if d['external_module']]:
mod_name = dep['full_mod_name']
metadata = dep['external_module_metadata']
self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
names = metadata.get('name', [])
versions = metadata.get('version', [None] * len(names))
self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s",
mod_name, names, versions)
for name, version in zip(names, versions):
self._simulated_load_dependency_module(name, version, metadata, verbose=True)
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname,
silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(
tcdep['name'])
self._simulated_load_dependency_module(
tcdep['name'], tcdep['version'],
{'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod,
silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version,
{'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(
os.path.join(install_path('mod'), mod_path_suffix,
modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def apply_regex_substitutions(path, regex_subs):
"""
Apply specified list of regex substitutions.
@param path: path to file to patch
@param regex_subs: list of substitutions to apply, specified as (<regexp pattern>, <replacement string>)
"""
# only report when in 'dry run' mode
if build_option('extended_dry_run'):
dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent'))
for regex, subtxt in regex_subs:
dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt))
else:
_log.debug("Applying following regex substitutions to %s: %s", path, regex_subs)
for i, (regex, subtxt) in enumerate(regex_subs):
regex_subs[i] = (re.compile(regex), subtxt)
for line in fileinput.input(path, inplace=1, backup='.orig.eb'):
for regex, subtxt in regex_subs:
line = regex.sub(subtxt, line)
sys.stdout.write(line)
def patch_perl_script_autoflush(path):
# patch Perl script to enable autoflush,
# so that e.g. run_cmd_qa receives all output to answer questions
# only report when in 'dry run' mode
if build_option('extended_dry_run'):
dry_run_msg("Perl script patched: %s" % path, silent=build_option('silent'))
else:
txt = read_file(path)
origpath = "%s.eb.orig" % path
write_file(origpath, txt)
_log.debug("Patching Perl script %s for autoflush, original script copied to %s" % (path, origpath))
# force autoflush for Perl print buffer
lines = txt.split('\n')
newtxt = '\n'.join([
lines[0], # shebang line
"\nuse IO::Handle qw();",
"STDOUT->autoflush(1);\n", # extra newline to separate from actual script
] + lines[1:])
write_file(path, newtxt)
def patch_perl_script_autoflush(path):
# patch Perl script to enable autoflush,
# so that e.g. run_cmd_qa receives all output to answer questions
# only report when in 'dry run' mode
if build_option('extended_dry_run'):
dry_run_msg("Perl script patched: %s" % path, silent=build_option('silent'))
else:
txt = read_file(path)
origpath = "%s.eb.orig" % path
write_file(origpath, txt)
_log.debug("Patching Perl script %s for autoflush, original script copied to %s" % (path, origpath))
# force autoflush for Perl print buffer
lines = txt.split('\n')
newtxt = '\n'.join([
lines[0], # shebang line
"\nuse IO::Handle qw();",
"STDOUT->autoflush(1);\n", # extra newline to separate from actual script
] + lines[1:])
write_file(path, newtxt)
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
maxhits=50, trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" % cmd)
# Part 1: process the QandA dictionary
# given initial set of Q and A (in dict), return dict of reg. exp. and A
#
# make regular expression that matches the string with
# - replace whitespace
# - replace newline
def escape_special(string):
return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
split = '[\s\n]+'
regSplit = re.compile(r"" + split)
def process_QA(q, a_s):
splitq = [escape_special(x) for x in regSplit.split(q)]
regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
# add optional split at the end
for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
a_s[i] += '\n'
regQ = re.compile(r"" + regQtxt)
if regQ.search(q):
return (a_s, regQ)
else:
raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt)
def check_answers_list(answers):
"""Make sure we have a list of answers (as strings)."""
if isinstance(answers, basestring):
answers = [answers]
elif not isinstance(answers, list):
raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)",
question, type(answers), answers)
# list is manipulated when answering matching question, so return a copy
return answers[:]
new_qa = {}
_log.debug("new_qa: ")
for question, answers in qa.items():
answers = check_answers_list(answers)
(answers, regQ) = process_QA(question, answers)
new_qa[regQ] = answers
_log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
new_std_qa = {}
if std_qa:
for question, answers in std_qa.items():
regQ = re.compile(r"" + question + r"[\s\n]*$")
answers = check_answers_list(answers)
for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]:
answers[i] += '\n'
new_std_qa[regQ] = answers
_log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
new_no_qa = []
if no_qa:
# simple statements, can contain wildcards
new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
_log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
# Part 2: Run the command and answer questions
# - this needs asynchronous stdout
# # Log command output
if cmd_log:
cmd_log.write("# output for interactive command: %s\n\n" % cmd)
try:
p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, stdin=PIPE, close_fds=True, executable="/bin/bash")
except OSError as err:
raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
ec = p.poll()
stdout_err = ''
old_len_out = -1
hit_count = 0
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
try:
out = recv_some(p)
if cmd_log:
cmd_log.write(out)
stdout_err += out
# recv_some may throw Exception
except (IOError, Exception) as err:
_log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
out = None
hit = False
for question, answers in new_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
_log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:])
send_all(p, fa)
hit = True
break
if not hit:
for question, answers in new_std_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
_log.debug("run_cmd_qa answer %s std question %s out %s", fa, question.pattern, stdout_err[-50:])
send_all(p, fa)
hit = True
break
if not hit:
if len(stdout_err) > old_len_out:
old_len_out = len(stdout_err)
else:
noqa = False
for r in new_no_qa:
if r.search(stdout_err):
_log.debug("runqanda: noQandA found for out %s", stdout_err[-50:])
noqa = True
if not noqa:
hit_count += 1
else:
hit_count = 0
else:
hit_count = 0
if hit_count > maxhits:
# explicitly kill the child process before exiting
try:
os.killpg(p.pid, signal.SIGKILL)
os.kill(p.pid, signal.SIGKILL)
except OSError as err:
_log.debug("run_cmd_qa exception caught when killing child process: %s", err)
_log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
cmd, maxhits, stdout_err[-500:])
# the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
time.sleep(1)
ec = p.poll()
# Process stopped. Read all remaining data
try:
if p.stdout:
out = p.stdout.read()
stdout_err += out
if cmd_log:
cmd_log.write(out)
cmd_log.close()
except IOError as err:
_log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
if trace:
trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp)
def run_cmd_qa(cmd,
qa,
no_qa=None,
log_ok=True,
log_all=False,
simple=False,
regexp=True,
std_qa=None,
path=None,
maxhits=50,
trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of interactive command: %s",
err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" %
cmd)
# Part 1: process the QandA dictionary
# given initial set of Q and A (in dict), return dict of reg. exp. and A
#
# make regular expression that matches the string with
# - replace whitespace
# - replace newline
def escape_special(string):
return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
split = r'[\s\n]+'
regSplit = re.compile(r"" + split)
def process_QA(q, a_s):
splitq = [escape_special(x) for x in regSplit.split(q)]
regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
# add optional split at the end
for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
a_s[i] += '\n'
regQ = re.compile(r"" + regQtxt)
if regQ.search(q):
return (a_s, regQ)
else:
raise EasyBuildError(
"runqanda: Question %s converted in %s does not match itself",
q, regQtxt)
def check_answers_list(answers):
"""Make sure we have a list of answers (as strings)."""
if isinstance(answers, string_type):
answers = [answers]
elif not isinstance(answers, list):
raise EasyBuildError(
"Invalid type for answer on %s, no string or list: %s (%s)",
question, type(answers), answers)
# list is manipulated when answering matching question, so return a copy
return answers[:]
new_qa = {}
_log.debug("new_qa: ")
for question, answers in qa.items():
answers = check_answers_list(answers)
(answers, regQ) = process_QA(question, answers)
new_qa[regQ] = answers
_log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
new_std_qa = {}
if std_qa:
for question, answers in std_qa.items():
regQ = re.compile(r"" + question + r"[\s\n]*$")
answers = check_answers_list(answers)
for i in [
idx for idx, a in enumerate(answers)
if not a.endswith('\n')
]:
answers[i] += '\n'
new_std_qa[regQ] = answers
_log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
new_no_qa = []
if no_qa:
# simple statements, can contain wildcards
new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
_log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
# Part 2: Run the command and answer questions
# - this needs asynchronous stdout
# # Log command output
if cmd_log:
cmd_log.write("# output for interactive command: %s\n\n" % cmd)
try:
proc = asyncprocess.Popen(cmd,
shell=True,
stdout=asyncprocess.PIPE,
stderr=asyncprocess.STDOUT,
stdin=asyncprocess.PIPE,
close_fds=True,
executable='/bin/bash')
except OSError as err:
raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
ec = proc.poll()
stdout_err = ''
old_len_out = -1
hit_count = 0
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
try:
out = get_output_from_process(proc, asynchronous=True)
if cmd_log:
cmd_log.write(out)
stdout_err += out
# recv_some used by get_output_from_process for getting asynchronous output may throw exception
except (IOError, Exception) as err:
_log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
out = None
hit = False
for question, answers in new_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug("List of answers for question %s after cycling: %s",
question.pattern, answers)
_log.debug("run_cmd_qa answer %s question %s out %s", fa,
question.pattern, stdout_err[-50:])
asyncprocess.send_all(proc, fa)
hit = True
break
if not hit:
for question, answers in new_std_qa.items():
res = question.search(stdout_err)
if out and res:
fa = answers[0] % res.groupdict()
# cycle through list of answers
last_answer = answers.pop(0)
answers.append(last_answer)
_log.debug(
"List of answers for question %s after cycling: %s",
question.pattern, answers)
_log.debug("run_cmd_qa answer %s std question %s out %s",
fa, question.pattern, stdout_err[-50:])
asyncprocess.send_all(proc, fa)
hit = True
break
if not hit:
if len(stdout_err) > old_len_out:
old_len_out = len(stdout_err)
else:
noqa = False
for r in new_no_qa:
if r.search(stdout_err):
_log.debug("runqanda: noQandA found for out %s",
stdout_err[-50:])
noqa = True
if not noqa:
hit_count += 1
else:
hit_count = 0
else:
hit_count = 0
if hit_count > maxhits:
# explicitly kill the child process before exiting
try:
os.killpg(proc.pid, signal.SIGKILL)
os.kill(proc.pid, signal.SIGKILL)
except OSError as err:
_log.debug(
"run_cmd_qa exception caught when killing child process: %s",
err)
_log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
raise EasyBuildError(
"run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
cmd, maxhits, stdout_err[-500:])
# the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
time.sleep(1)
ec = proc.poll()
# Process stopped. Read all remaining data
try:
if proc.stdout:
out = get_output_from_process(proc)
stdout_err += out
if cmd_log:
cmd_log.write(out)
cmd_log.close()
except IOError as err:
_log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
if trace:
trace_msg("interactive command completed: exit %s, ran in %s" %
(ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError(
"Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok,
regexp)
def run_cmd_qa(cmd,
qa,
no_qa=None,
log_ok=True,
log_all=False,
simple=False,
regexp=True,
std_qa=None,
path=None,
maxhits=50,
trace=True):
"""
Run specified interactive command (in a subshell)
:param cmd: command to run
:param qa: dictionary which maps question to answers
:param no_qa: list of patters that are not questions
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param std_qa: dictionary which maps question regex patterns to answers
:param path: path to execute the command is; current working directory is used if unspecified
:param maxhits: maximum number of cycles (seconds) without being able to find a known question
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if log_all or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd_qa-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of interactive command: %s",
err)
_log.debug('run_cmd_qa: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running interactive command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd.strip())
# early exit in 'dry run' mode, after printing the command that would be run
if build_option('extended_dry_run'):
if path is None:
path = cwd
dry_run_msg(" running interactive command \"%s\"" % cmd,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!" %
cmd)
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=True,
trace=True,
stream_output=None):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
"""
cwd = os.getcwd()
if isinstance(cmd, string_type):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of command: %s",
err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info(
"Auto-enabling streaming output of '%s' command because logging to stdout is enabled",
cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
if inp:
trace_txt += "\t[input: %s]\n" % inp
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
if cmd_log:
cmd_log.write("# output for command: %s\n\n" % cmd_msg)
exec_cmd = "/bin/bash"
if not shell:
if isinstance(cmd, list):
exec_cmd = None
cmd.insert(0, '/usr/bin/env')
elif isinstance(cmd, string_type):
cmd = '/usr/bin/env %s' % cmd
else:
raise EasyBuildError(
"Don't know how to prefix with /usr/bin/env for commands of type %s",
type(cmd))
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd,
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
executable=exec_cmd)
except OSError as err:
raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
if inp:
proc.stdin.write(inp.encode())
proc.stdin.close()
# use small read size when streaming output, to make it stream more fluently
# read size should not be too small though, to avoid too much overhead
if stream_output:
read_size = 128
else:
read_size = 1024 * 8
ec = proc.poll()
stdouterr = ''
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
output = get_output_from_process(proc, read_size=read_size)
if cmd_log:
cmd_log.write(output)
if stream_output:
sys.stdout.write(output)
stdouterr += output
ec = proc.poll()
# read remaining data (all of it)
output = get_output_from_process(proc)
if cmd_log:
cmd_log.write(output)
cmd_log.close()
if stream_output:
sys.stdout.write(output)
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" %
(ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError(
"Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok,
regexp)
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=None,
trace=True,
stream_output=None,
asynchronous=False):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
:param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
"""
cwd = os.getcwd()
if isinstance(cmd, string_type):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if shell is None:
shell = True
if isinstance(cmd, list):
raise EasyBuildError(
"When passing cmd as a list then `shell` must be set explictely! "
"Note that all elements of the list but the first are treated as arguments "
"to the shell and NOT to the command to be executed!")
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of command: %s",
err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info(
"Auto-enabling streaming output of '%s' command because logging to stdout is enabled",
cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
if inp:
trace_txt += "\t[input: %s]\n" % inp
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
if cmd_log:
cmd_log.write("# output for command: %s\n\n" % cmd_msg)
exec_cmd = "/bin/bash"
if not shell:
if isinstance(cmd, list):
exec_cmd = None
cmd.insert(0, '/usr/bin/env')
elif isinstance(cmd, string_type):
cmd = '/usr/bin/env %s' % cmd
else:
raise EasyBuildError(
"Don't know how to prefix with /usr/bin/env for commands of type %s",
type(cmd))
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd,
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
executable=exec_cmd)
except OSError as err:
raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
if inp:
proc.stdin.write(inp.encode())
proc.stdin.close()
if asynchronous:
return (proc, cmd, cwd, start_time, cmd_log)
else:
return complete_cmd(proc,
cmd,
cwd,
start_time,
cmd_log,
log_ok=log_ok,
log_all=log_all,
simple=simple,
regexp=regexp,
stream_output=stream_output,
trace=trace)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
"""
cwd = os.getcwd()
if isinstance(cmd, basestring):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
"""
cwd = os.getcwd()
if isinstance(cmd, basestring):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
# auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
if stream_output is None and build_option('logtostdout'):
_log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
stream_output = True
if stream_output:
print_msg("(streaming) output for command '%s':" % cmd_msg)
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError as err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)
if cmd_log:
cmd_log.write("# output for command: %s\n\n" % cmd_msg)
exec_cmd = "/bin/bash"
if not shell:
if isinstance(cmd, list):
exec_cmd = None
cmd.insert(0, '/usr/bin/env')
elif isinstance(cmd, basestring):
cmd = '/usr/bin/env %s' % cmd
else:
raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd)
except OSError as err:
raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
if inp:
proc.stdin.write(inp)
proc.stdin.close()
# use small read size when streaming output, to make it stream more fluently
# read size should not be too small though, to avoid too much overhead
if stream_output:
read_size = 128
else:
read_size = 1024 * 8
ec = proc.poll()
stdouterr = ''
while ec is None:
# need to read from time to time.
# - otherwise the stdout/stderr buffer gets filled and it all stops working
output = proc.stdout.read(read_size)
if cmd_log:
cmd_log.write(output)
if stream_output:
sys.stdout.write(output)
stdouterr += output
ec = proc.poll()
# read remaining data (all of it)
output = proc.stdout.read()
if cmd_log:
cmd_log.write(output)
cmd_log.close()
if stream_output:
sys.stdout.write(output)
stdouterr += output
if trace:
trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
try:
os.chdir(cwd)
except OSError as err:
raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
def run_cmd(cmd,
log_ok=True,
log_all=False,
simple=False,
inp=None,
regexp=True,
log_output=False,
path=None,
force_in_dry_run=False,
verbose=True,
shell=True,
trace=True):
"""
Run specified command (in a subshell)
:param cmd: command to run
:param log_ok: only run output/exit code for failing commands (exit code non-zero)
:param log_all: always log command output and exit code
:param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
:param inp: the input given to the command via stdin
:param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param log_output: indicate whether all output of command should be logged to a separate temporary logfile
:param path: path to execute the command in; current working directory is used if unspecified
:param force_in_dry_run: force running the command during dry run
:param verbose: include message on running the command in dry run output
:param shell: allow commands to not run in a shell (especially useful for cmd lists)
:param trace: print command being executed as part of trace output
"""
cwd = os.getcwd()
if isinstance(cmd, basestring):
cmd_msg = cmd.strip()
elif isinstance(cmd, list):
cmd_msg = ' '.join(cmd)
else:
raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
if log_output or (trace and build_option('trace')):
# collect output of running command in temporary log file, if desired
fd, cmd_log_fn = tempfile.mkstemp(suffix='.log',
prefix='easybuild-run_cmd-')
os.close(fd)
try:
cmd_log = open(cmd_log_fn, 'w')
except IOError as err:
raise EasyBuildError(
"Failed to open temporary log file for output of command: %s",
err)
_log.debug('run_cmd: Output of "%s" will be logged to %s' %
(cmd, cmd_log_fn))
else:
cmd_log_fn, cmd_log = None, None
start_time = datetime.now()
if trace:
trace_txt = "running command:\n"
trace_txt += "\t[started at: %s]\n" % start_time.strftime(
'%Y-%m-%d %H:%M:%S')
trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
trace_msg(trace_txt + '\t' + cmd_msg)
# early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
if not force_in_dry_run and build_option('extended_dry_run'):
if path is None:
path = cwd
if verbose:
dry_run_msg(" running command \"%s\"" % cmd_msg,
silent=build_option('silent'))
dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
# make sure we get the type of the return value right
if simple:
return True
else:
# output, exit code
return ('', 0)
try:
if path:
os.chdir(path)
_log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
except OSError, err:
_log.warning("Failed to change to %s: %s" % (path, err))
_log.info("running cmd %s in non-existing directory, might fail!", cmd)