def __init__(self):
# Try to figure out if we are indeed using the TCL version
try:
completed = os_ext.run_command('modulecmd -V')
except OSError as e:
raise ReframeError('could not find a sane Tmod installation: %s' %
e)
version_match = re.search(r'^VERSION=(\S+)', completed.stdout,
re.MULTILINE)
tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout,
re.MULTILINE)
if version_match is None or tcl_version_match is None:
raise ReframeError('could not find a sane Tmod installation')
self._version = version_match.group(1)
self._command = 'modulecmd python'
try:
# Try the Python bindings now
completed = os_ext.run_command(self._command)
except OSError as e:
raise ReframeError('could not get the Python bindings for Tmod: ' %
e)
if re.search(r'Unknown shell type', completed.stderr):
raise ReframeError(
'Python is not supported by this Tmod installation')
def standard_threshold(value, reference, logger=None):
try:
refval, thres_lower, thres_upper = reference
except (ValueError, TypeError):
raise ReframeError('Improper reference value')
if logger:
logger.info('value: %s, reference: %s' % (str(value), reference))
# sanity checking of user input
if refval is None:
raise ReframeError(
'No reference value specified for calculating tolerance')
if thres_lower is None and thres_upper is None:
return True
if thres_lower is None:
_expect_interval(thres_upper, (0, 1), 'reference upper threshold')
return value <= _bound(refval, thres_upper)
if thres_upper is None:
_expect_interval(thres_lower, (-1, 0), 'reference lower threshold')
return value >= _bound(refval, thres_lower)
_expect_interval(thres_upper, (0, 1), 'reference upper threshold')
_expect_interval(thres_lower, (-1, 0), 'reference lower threshold')
return (value >= _bound(refval, thres_lower)
and value <= _bound(refval, thres_upper))
def copytree_virtual(src, dst, file_links=[],
symlinks=False, copy_function=shutil.copy2,
ignore_dangling_symlinks=False):
"""Copy `dst` to `src`, but create symlinks for the files in `file_links`.
If `file_links` is empty, this is equivalent to `copytree()`. The rest of
the arguments are passed as-is to `copytree()`. Paths in `file_links` must
be relative to `src`. If you try to pass `.` in `file_links`, `OSError`
will be raised."""
if not hasattr(file_links, '__iter__'):
raise TypeError('expecting an iterable as file_links')
# Work with absolute paths
src = os.path.abspath(src)
dst = os.path.abspath(dst)
# 1. Check that the link targets are valid
# 2. Convert link targes to absolute paths
# 3. Store them in a set for quick look up inside the ignore function
link_targets = set()
for f in file_links:
if os.path.isabs(f):
raise ReframeError(
"copytree_virtual() failed: `%s': "
"absolute paths not allowed in file_links" % f)
target = os.path.join(src, f)
if not os.path.exists(target):
raise ReframeError(
"copytree_virtual() failed: `%s' does not exist" % target)
if os.path.commonpath([src, target]) != src:
raise ReframeError(
"copytree_virtual() failed: "
"`%s' not under `%s'" % (target, src))
link_targets.add(os.path.abspath(target))
if not file_links:
ignore = None
else:
def ignore(dir, contents):
return [c for c in contents
if os.path.join(dir, c) in link_targets]
# Copy to dst ignoring the file_links
copytree(src, dst, symlinks, ignore,
copy_function, ignore_dangling_symlinks)
# Now create the symlinks
for f in link_targets:
link_name = f.replace(src, dst)
os.symlink(f, link_name)
def git_clone(url, targetdir=None):
'''Clone git repository from a URL.'''
if not git_repo_exists(url):
raise ReframeError('git repository does not exist')
targetdir = targetdir or ''
run_command('git clone %s %s' % (url, targetdir), check=True)
def wait(self):
"""Wait for the spawned job to finish.
As soon as the parent job process finishes, all of its spawned
subprocesses will be forced to finish, too.
Upon return, the whole process tree of the spawned job process will be
cleared, unless any of them has called `setsid()`.
"""
if self._jobid is None:
raise ReframeError('no job is spawned yet')
if self._state is not None:
# Job has been already waited for
return
# Convert job's time_limit to seconds
h, m, s = self.time_limit
timeout = h * 3600 + m * 60 + s
try:
self._wait_all(timeout)
self._exitcode = self._proc.returncode
if self._exitcode != 0:
self._state = LOCAL_JOB_FAILURE
else:
self._state = LOCAL_JOB_SUCCESS
except (_TimeoutExpired, subprocess.TimeoutExpired):
getlogger().debug('job timed out')
self._state = LOCAL_JOB_TIMEOUT
finally:
# Cleanup all the processes of this job
self._kill_all()
self._wait_all()
self._f_stdout.close()
self._f_stderr.close()
def _expect_interval(val, interval, valdescr=None):
lower, upper = interval
if val < lower or val > upper:
if not valdescr:
valdescr = 'value'
raise ReframeError('%s (%s) not in [%s,%s]' %
(valdescr, val, lower, upper))
def init_modules_system(modules_kind=None):
global _modules_system
if modules_kind is None:
_modules_system = ModulesSystem(NoModImpl())
elif modules_kind == 'tmod':
_modules_system = ModulesSystem(TModImpl())
else:
raise ReframeError('unknown module system')
def cancel(self):
getlogger().debug('cancelling job (id=%s)' % self._jobid)
if self._jobid is None:
raise ReframeError('no job is spawned yet')
os_ext.run_command('scancel %s' % self._jobid,
check=True,
timeout=settings.job_submit_timeout)
self._is_cancelling = True
def load_from_file(self, filename, **check_args):
module_name = self._module_name(filename)
try:
if not self._validate_source(filename):
return []
loader = SourceFileLoader(module_name, filename)
return self.load_from_module(loader.load_module(), **check_args)
except OSError as e:
raise ReframeError(
"Could not load module `%s' from file `%s': %s" %
(module_name, filename, e.strerror))
def test_logging_context_error(default_exec_ctx, logfile):
rlog.configure_logging(rt.runtime().site_config)
try:
with rlog.logging_context(level=rlog.ERROR):
raise ReframeError('error from context')
pytest.fail('logging_context did not propagate the exception')
except ReframeError:
pass
assert _found_in_logfile('reframe', logfile)
assert _found_in_logfile('error from context', logfile)
def test_logging_context_error(self):
configure_logging(self.logging_config)
try:
with logging_context(level=ERROR):
raise ReframeError('error from context')
self.fail('logging_context did not propagate the exception')
except ReframeError:
pass
self.assertTrue(self.found_in_logfile('reframe'))
self.assertTrue(self.found_in_logfile('error from context'))
def test_logging_context_error(self):
rlog.configure_logging(self.logging_config)
try:
with rlog.logging_context(level=rlog.ERROR):
raise ReframeError('error from context')
pytest.fail('logging_context did not propagate the exception')
except ReframeError:
pass
assert self.found_in_logfile('reframe')
assert self.found_in_logfile('error from context')
def wait(self):
if self._jobid is None:
raise ReframeError('no job is spawned yet')
# Quickly return in case we have finished already
if self._state in self._completion_states:
return
intervals = itertools.cycle(settings.job_poll_intervals)
self._update_state()
while self._state not in self._completion_states:
time.sleep(next(intervals))
self._update_state()
def _check_level(level):
if isinstance(level, numbers.Integral):
ret = level
elif isinstance(level, str):
norm_level = level.lower()
if norm_level not in _log_level_values:
raise ReframeError('logger level %s not available' % level)
else:
ret = _log_level_values[norm_level]
else:
raise TypeError('logger level %s not an int or a valid string' % level)
return ret
def git_clone(url, targetdir=None):
'''Clone a git repository from a URL.
:arg url: The URL to clone from.
:arg targetdir: The directory where the repository will be cloned to. If
:class:`None`, a new directory will be created with the repository
name as if ``git clone {url}`` was issued.
'''
if not git_repo_exists(url):
raise ReframeError('git repository does not exist')
targetdir = targetdir or ''
run_command('git clone %s %s' % (url, targetdir), check=True)
def git_clone(url, targetdir=None, opts=None, timeout=5):
'''Clone a git repository from a URL.
:arg url: The URL to clone from.
:arg opts: List of options to be passed to the `git clone` command
:arg timeout: Timeout in seconds when checking if the url is a valid
repository.
:arg targetdir: The directory where the repository will be cloned to. If
:class:`None`, a new directory will be created with the repository
name as if ``git clone {url}`` was issued.
'''
if not git_repo_exists(url, timeout=timeout):
raise ReframeError('git repository does not exist')
targetdir = targetdir or ''
opts = ' '.join(opts) if opts is not None else ''
run_command(f'git clone {opts} {url} {targetdir}', check=True)
def cancel(self):
"""Cancel job.
The SIGTERM signal will be sent first to all the processes of this job
and after a grace period (default 2s) the SIGKILL signal will be send.
This function waits for the spawned process tree to finish.
"""
if self._jobid is None:
raise ReframeError('no job is spawned yet')
self._term_all()
# Set the time limit to the grace period and let wait() do the final
# killing
self._time_limit = (0, 0, self.cancel_grace_period)
self.wait()
def _compile_file(self, source_file, executable, lang, options):
if not executable:
# default executable, same as source_file without the extension
executable = os.path.join(os.path.dirname(source_file),
source_file.rsplit('.')[:-1][0])
if not lang:
lang = self.guess_language(source_file)
# Replace None's with empty strings
cppflags = self.cppflags or ''
cflags = self.cflags or ''
cxxflags = self.cxxflags or ''
fflags = self.fflags or ''
ldflags = self.ldflags or ''
flags = [cppflags]
if lang == 'C':
compiler = self.cc
flags.append(cflags)
elif lang == 'C++':
compiler = self.cxx
flags.append(cxxflags)
elif lang == 'Fortran':
compiler = self.ftn
flags.append(fflags)
elif lang == 'CUDA':
compiler = 'nvcc'
flags.append(cxxflags)
else:
raise ReframeError('Unknown language')
# Append include search path
flags += ['-I' + d for d in self.include_search_path]
cmd = ('%s %s %s -o %s %s %s' % (compiler, ' '.join(flags),
source_file, executable,
ldflags, options))
try:
return os_ext.run_command(cmd, check=True)
except CommandError as e:
raise CompilationError(command=e.command,
stdout=e.stdout,
stderr=e.stderr,
exitcode=e.exitcode,
environ=self)
def setup(self, system, environ, **job_opts):
raise ReframeError('Setup failure')
def raise_error_early(self):
raise ReframeError('Setup failure')
def get_modules_system():
if _modules_system is None:
raise ReframeError('no modules system is configured')
return _modules_system
def re_compile(patt):
try:
return re.compile(patt)
except re.error:
raise ReframeError(f'invalid regex: {patt!r}')
def re_compile(patt):
try:
return re.compile(patt)
except re.error:
raise ReframeError("invalid regex: '%s'" % patt)