def check_performance(self):
"""The performance checking phase of the regression test pipeline.
:raises reframe.core.exceptions.SanityError: If the performance check
fails.
"""
if self.perf_patterns is None:
return
with os_ext.change_dir(self._stagedir):
# We first evaluate and log all performance values and then we
# check them against the reference. This way we always log them
# even if the don't meet the reference.
perf_values = []
for tag, expr in self.perf_patterns.items():
value = evaluate(expr)
key = '%s:%s' % (self._current_partition.fullname, tag)
if key not in self.reference:
raise SanityError(
"tag `%s' not resolved in references for `%s'" %
(tag, self._current_partition.fullname))
perf_values.append((value, self.reference[key]))
self._perf_logger.log_performance(logging.INFO, tag, value,
*self.reference[key])
for val, reference in perf_values:
ref, low_thres, high_thres, *_ = reference
evaluate(assert_reference(val, ref, low_thres, high_thres))
def keep_files_list(self, test, compile_only=False):
from reframe.core.deferrable import evaluate
ret = [self.replace_prefix(evaluate(test.stdout), test.outputdir),
self.replace_prefix(evaluate(test.stderr), test.outputdir)]
if not compile_only:
ret.append(self.replace_prefix(test.job.script_filename,
test.outputdir))
ret.extend([self.replace_prefix(f, test.outputdir)
for f in test.keep_files])
return ret
def test_avg(self):
res = evaluate(avg([1, 2, 3, 4]))
self.assertEqual(2.5, res)
# Check result when passing a generator
res = evaluate(avg(range(1, 5)))
self.assertEqual(2.5, res)
# Check with single element container
res = evaluate(avg(range(1, 2)))
self.assertEqual(1, res)
# Check with empty container
self.assertRaises(SanityError, evaluate, avg([]))
def extractsingle(patt, filename, tag=0, conv=None, item=0, encoding='utf-8'):
"""Extract a single value from the capturing group ``tag`` of a matching regex
``patt`` in the file ``filename``.
This function is equivalent to ``extractall(patt, filename, tag,
conv)[item]``, except that it raises a ``SanityError`` if ``item`` is out of
bounds.
:arg patt: as in :func:`extractall`.
:arg filename: as in :func:`extractall`.
:arg encoding: as in :func:`extractall`.
:arg tag: as in :func:`extractall`.
:arg conv: as in :func:`extractall`.
:arg item: the specific element to extract.
:returns: The extracted value.
:raises reframe.core.exceptions.SanityError: In case of errors.
"""
try:
# Explicitly evaluate the expression here, so as to force any exception
# to be thrown in this context and not during the evaluation of an
# expression containing this one.
return evaluate(extractall(patt, filename, tag, conv, encoding)[item])
except IndexError:
raise SanityError("not enough matches of pattern `%s' in file `%s' "
"so as to extract item `%s'" %
(patt, filename, item))
def assert_not_found(patt, filename, msg=None, encoding='utf-8'):
"""Assert that regex pattern ``patt`` is not found in the file
``filename``.
This is the inverse of :func:`assert_found()`.
:returns: ``True`` on success.
:raises reframe.core.exceptions.SanityError: if assertion fails.
"""
try:
evaluate(assert_found(patt, filename, msg, encoding))
except SanityError:
return True
else:
error_msg = msg or "pattern `{0}' found in `{1}'"
raise SanityError(_format(error_msg, patt, filename))
def test_extractall(self):
# Check numeric groups
res = evaluate(extractall('Step: (?P<no>\d+)', self.tempfile, 1))
for expected, v in builtins.enumerate(res, start=1):
self.assertEqual(str(expected), v)
# Check named groups
res = evaluate(extractall('Step: (?P<no>\d+)', self.tempfile, 'no'))
for expected, v in builtins.enumerate(res, start=1):
self.assertEqual(str(expected), v)
# Check convert function
res = evaluate(
extractall('Step: (?P<no>\d+)', self.tempfile, 'no', builtins.int))
for expected, v in builtins.enumerate(res, start=1):
self.assertEqual(expected, v)
def test_chain(self):
list1 = ['A', 'B', 'C']
list2 = ['D', 'E', 'F']
chain1 = evaluate(chain(make_deferrable(list1), list2))
chain2 = itertools.chain(list1, list2)
self.assertTrue(
builtins.all((a == b for a, b in builtins.zip(chain1, chain2))))
def test_map(self):
l = [1, 2, 3]
dm = map(lambda x: 2 * x + 1, l)
for i, x in enumerate(dm, start=1):
self.assertEqual(2 * i + 1, x)
# Alternative test
self.assertEqual([3, 5, 7], list(evaluate(dm)))
def test_filter(self):
df = filter(lambda x: x if x % 2 else None,
make_deferrable([1, 2, 3, 4, 5]))
for i, x in enumerate(df, start=1):
self.assertEqual(2 * i - 1, x)
# Alternative testing
self.assertEqual([1, 3, 5], list(evaluate(df)))
def test_findall(self):
res = evaluate(findall('Step: \d+', self.tempfile))
self.assertEqual(3, builtins.len(res))
res = evaluate(findall('Step:.*', self.tempfile))
self.assertEqual(3, builtins.len(res))
res = evaluate(findall('Step: [12]', self.tempfile))
self.assertEqual(2, builtins.len(res))
# Check the matches
for expected, match in builtins.zip(['Step: 1', 'Step: 2'], res):
self.assertEqual(expected, match.group(0))
# Check groups
res = evaluate(findall('Step: (?P<no>\d+)', self.tempfile))
for step, match in builtins.enumerate(res, start=1):
self.assertEqual(step, builtins.int(match.group(1)))
self.assertEqual(step, builtins.int(match.group('no')))
def check_sanity(self):
"""The sanity checking phase of the regression test pipeline.
:raises reframe.core.exceptions.SanityError: If the sanity check fails.
"""
if self.sanity_patterns is None:
raise SanityError('sanity_patterns not set')
with os_ext.change_dir(self._stagedir):
success = evaluate(self.sanity_patterns)
if not success:
raise SanityError()
def read_timestamps_sorted(self):
from reframe.core.deferrable import evaluate
self.begin_stamps = []
self.end_stamps = []
for c in self.debug_policy.checks:
with open(evaluate(c.stdout), 'r') as f:
self.begin_stamps.append(float(f.readline().strip()))
self.end_stamps.append(float(f.readline().strip()))
self.begin_stamps.sort()
self.end_stamps.sort()
def check_performance(self):
"""The performance checking phase of the regression test pipeline.
:raises reframe.core.exceptions.SanityError: If the performance check
fails.
"""
if self.perf_patterns is None:
return
with os_ext.change_dir(self._stagedir):
# We first evaluate and log all performance values and then we
# check them against the reference. This way we always log them
# even if the don't meet the reference.
for tag, expr in self.perf_patterns.items():
value = evaluate(expr)
key = '%s:%s' % (self._current_partition.fullname, tag)
if key not in self.reference:
raise SanityError(
"tag `%s' not resolved in references for `%s'" %
(tag, self._current_partition.fullname))
self._perfvalues[key] = (tag, value, *self.reference[key])
self._perf_logger.log_performance(logging.INFO, tag, value,
*self.reference[key])
for values in self._perfvalues.values():
tag, val, ref, low_thres, high_thres, *_ = values
try:
evaluate(
assert_reference(
val,
ref,
low_thres,
high_thres,
msg=('failed to meet reference: %s={0}, '
'expected {1} (l={2}, u={3})' % tag),
))
except SanityError as e:
raise PerformanceError(e)
def assert_found(patt, filename, msg=None, encoding='utf-8'):
"""Assert that regex pattern ``patt`` is found in the file ``filename``.
:arg patt: The regex pattern to search.
Any standard Python `regular expression
<https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`_
is accepted.
:arg filename: The name of the file to examine.
Any :class:`OSError` raised while processing the file will be
propagated as a :class:`reframe.core.exceptions.SanityError`.
:arg encoding: The name of the encoding used to decode the file.
:returns: ``True`` on success.
:raises reframe.core.exceptions.SanityError: if assertion fails.
"""
num_matches = count(finditer(patt, filename, encoding))
try:
evaluate(assert_true(num_matches))
except SanityError:
error_msg = msg or "pattern `{0}' not found in `{1}'"
raise SanityError(_format(error_msg, patt, filename))
else:
return True
def check_performance(self):
"""The performance checking phase of the regression test pipeline.
:raises reframe.core.exceptions.SanityError: If the performance check
fails.
"""
if self.perf_patterns is None:
return
with os_ext.change_dir(self._stagedir):
for tag, expr in self.perf_patterns.items():
value = evaluate(expr)
key = '%s:%s' % (self._current_partition.fullname, tag)
try:
ref, low_thres, high_thres = self.reference[key]
self._perf_logger.info('value: %s, reference: %s' %
(value, self.reference[key]))
except KeyError:
raise SanityError(
"tag `%s' not resolved in references for `%s'" %
(tag, self._current_partition.fullname))
evaluate(assert_reference(value, ref, low_thres, high_thres))
def do_sanity_check(self):
failures = []
devices_found = set(
sn.extractall(
r'^\s*([^,]*),\s*Detected devices: %s' %
self.num_gpus_per_node, self.stdout, 1))
evaluate(
sn.assert_eq(self.job.num_tasks,
len(devices_found),
msg='requested {0} node(s), got {1} (nodelist: %s)' %
','.join(sorted(devices_found))))
good_nodes = set(
sn.extractall(r'^\s*([^,]*),\s*NID\s*=\s*\S+\s+Result = PASS',
self.stdout, 1))
evaluate(
sn.assert_eq(devices_found,
good_nodes,
msg='check failed on the following node(s): %s' %
','.join(sorted(devices_found - good_nodes))))
# Sanity is fine, fill in the perf. patterns based on the exact node id
for nodename in devices_found:
for xfer_kind in ('h2d', 'd2h', 'd2d'):
for devno in range(self.num_gpus_per_node):
perfvar = '%s_gpu_%s_%s_bw' % (nodename, devno, xfer_kind)
perfvar = 'bw_%s_%s_gpu_%s' % (xfer_kind, nodename, devno)
self.perf_patterns[perfvar] = sn.extractsingle(
self._xfer_pattern(xfer_kind, devno, nodename),
self.stdout, 1, float, 0)
partname = self.current_partition.fullname
refkey = '%s:%s' % (partname, perfvar)
bwkey = '%s:%s' % (partname, xfer_kind)
self.reference[refkey] = self.__bwref[bwkey]
return True
def read_timestamps(self, tasks):
"""Read the timestamps and sort them to permit simple
concurrency tests."""
from reframe.core.deferrable import evaluate
self.begin_stamps = []
self.end_stamps = []
for t in tasks:
with open(evaluate(t.check.stdout), 'r') as f:
self.begin_stamps.append(float(f.readline().strip()))
self.end_stamps.append(float(f.readline().strip()))
self.begin_stamps.sort()
self.end_stamps.sort()
def findall(patt, filename, encoding='utf-8'):
"""Get all matches of regex ``patt`` in ``filename``.
:arg patt: The regex pattern to search.
Any standard Python `regular expression
<https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`_
is accepted.
:arg filename: The name of the file to examine.
:arg encoding: The name of the encoding used to decode the file.
:returns: A list of raw `regex match objects
<https://docs.python.org/3.6/library/re.html#match-objects>`_.
:raises reframe.core.exceptions.SanityError: In case an :class:`OSError` is
raised while processing ``filename``.
"""
return list(evaluate(x) for x in finditer(patt, filename, encoding))
def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None):
"""Assert that value ``val`` respects the reference value ``ref``.
:arg val: The value to check.
:arg ref: The reference value.
:arg lower_thres: The lower threshold value expressed as a negative decimal
fraction of the reference value. Must be in [-1, 0] for ref >= 0.0 and
in [-inf, 0] for ref < 0.0.
If ``None``, no lower thresholds is applied.
:arg upper_thres: The upper threshold value expressed as a decimal fraction
of the reference value. Must be in [0, inf] for ref >= 0.0 and
in [0, 1] for ref < 0.0.
If ``None``, no upper thresholds is applied.
:returns: ``True`` on success.
:raises reframe.core.exceptions.SanityError: if assertion fails or if the
lower and upper thresholds do not have appropriate values.
"""
if lower_thres is not None:
lower_thres_limit = -1 if ref >= 0 else None
try:
evaluate(assert_bounded(lower_thres, lower_thres_limit, 0))
except SanityError:
raise SanityError('invalid low threshold value: %s' %
lower_thres) from None
if upper_thres is not None:
upper_thres_limit = None if ref >= 0 else 1
try:
evaluate(assert_bounded(upper_thres, 0, upper_thres_limit))
except SanityError:
raise SanityError('invalid high threshold value: %s' %
upper_thres) from None
def calc_bound(thres):
if thres is None:
return None
# Inverse threshold if ref < 0
if ref < 0:
thres = -thres
return ref * (1 + thres)
lower = calc_bound(lower_thres) or float('-inf')
upper = calc_bound(upper_thres) or float('inf')
try:
evaluate(assert_bounded(val, lower, upper))
except SanityError:
error_msg = '{0} is beyond reference value {1} (l={2}, u={3})'
raise SanityError(_format(error_msg, val, ref, lower, upper))
else:
return True
def test_extractall_custom_conv(self):
res = evaluate(
sn.extractall('Step: (\d+)', self.tempfile, 1, lambda x: int(x)))
for expected, v in enumerate(res, start=1):
self.assertEqual(expected, v)
# Check error in custom function
self.assertRaises(
SanityError, evaluate,
sn.extractall('Step: (\d+)', self.tempfile, conv=lambda x: int(x)))
# Check error with a callable object
class C:
def __call__(self, x):
return int(x)
self.assertRaises(
SanityError, evaluate,
sn.extractall('Step: (\d+)', self.tempfile, conv=C()))
def extractall(patt, filename, tag=0, conv=None, encoding='utf-8'):
"""Extract all values from the capturing group ``tag`` of a matching regex
``patt`` in the file ``filename``.
:arg patt: The regex pattern to search.
Any standard Python `regular expression
<https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`_
is accepted.
:arg filename: The name of the file to examine.
:arg encoding: The name of the encoding used to decode the file.
:arg tag: The regex capturing group to be extracted.
Group ``0`` refers always to the whole match.
Since the file is processed line by line, this means that group ``0``
returns the whole line that was matched.
:arg conv: A callable that takes a single argument and returns a new value.
If provided, it will be used to convert the extracted values before
returning them.
:returns: A list of the extracted values from the matched regex.
:raises reframe.core.exceptions.SanityError: In case of errors.
"""
return list(
evaluate(x) for x in extractiter(patt, filename, tag, conv, encoding))
def check_performance(self):
"""The performance checking phase of the regression test pipeline.
:raises reframe.core.exceptions.SanityError: If the performance check
fails.
"""
if self.perf_patterns is None:
return
with os_ext.change_dir(self._stagedir):
# Check if default reference perf values are provided and
# store all the variables tested in the performance check
has_default = False
variables = set()
for key, ref in self.reference.items():
keyparts = key.split(self.reference.scope_separator)
system = keyparts[0]
varname = keyparts[-1]
try:
unit = ref[3]
except IndexError:
unit = None
variables.add((varname, unit))
if system == '*':
has_default = True
break
if not has_default:
if not variables:
# If empty, it means that self.reference was empty, so try
# to infer their name from perf_patterns
variables = {(name, None)
for name in self.perf_patterns.keys()}
for var in variables:
name, unit = var
ref_tuple = (0, None, None)
if unit:
ref_tuple += (unit, )
self.reference.update({'*': {name: ref_tuple}})
# We first evaluate and log all performance values and then we
# check them against the reference. This way we always log them
# even if the don't meet the reference.
for tag, expr in self.perf_patterns.items():
value = evaluate(expr)
key = '%s:%s' % (self._current_partition.fullname, tag)
if key not in self.reference:
raise SanityError(
"tag `%s' not resolved in references for `%s'" %
(tag, self._current_partition.fullname))
self._perfvalues[key] = (value, *self.reference[key])
self._perf_logger.log_performance(logging.INFO, tag, value,
*self.reference[key])
for key, values in self._perfvalues.items():
val, ref, low_thres, high_thres, *_ = values
tag = key.split(':')[-1]
try:
evaluate(
assert_reference(
val,
ref,
low_thres,
high_thres,
msg=('failed to meet reference: %s={0}, '
'expected {1} (l={2}, u={3})' % tag)))
except SanityError as e:
raise PerformanceError(e)
def test_ixor(self):
v = V(7)
dv = make_deferrable(v)
dv ^= V(7)
evaluate(dv)
self.assertEqual(0, v._value)
def test_ior(self):
v = V(2)
dv = make_deferrable(v)
dv |= V(5)
evaluate(dv)
self.assertEqual(7, v._value)
def test_extractsingle_encoding(self):
res = evaluate(
extractsingle(r'Odyssey', self.utf16_file, encoding='utf-16'))
self.assertNotEqual(-1, res.find('Odyssey'))
def test_extractall_encoding(self):
res = evaluate(
extractall(r'Odyssey', self.utf16_file, encoding='utf-16'))
self.assertEqual(1, len(res))
def test_findall_encoding(self):
res = evaluate(
sn.findall('Odyssey', self.utf16_file, encoding='utf-16'))
self.assertEqual(1, len(res))
def test_sorted(self):
l = [2, 3, 1]
ds = sorted(l)
self.assertEqual([1, 2, 3], list(evaluate(ds)))
def test_setattr(self):
dset = setattr(self, '_a', 5)
self.assertEqual(0, self._a)
evaluate(dset)
self.assertEqual(5, self._a)
def test_reversed(self):
l = [1, 2, 3]
dr = reversed(l)
self.assertEqual([3, 2, 1], list(evaluate(dr)))
