def check_performance(self): """The performance checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the performance check fails. """ if self.perf_patterns is None: return with os_ext.change_dir(self._stagedir): # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them # even if the don't meet the reference. perf_values = [] for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) if key not in self.reference: raise SanityError( "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) perf_values.append((value, self.reference[key])) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) for val, reference in perf_values: ref, low_thres, high_thres, *_ = reference evaluate(assert_reference(val, ref, low_thres, high_thres))
def keep_files_list(self, test, compile_only=False): from reframe.core.deferrable import evaluate ret = [self.replace_prefix(evaluate(test.stdout), test.outputdir), self.replace_prefix(evaluate(test.stderr), test.outputdir)] if not compile_only: ret.append(self.replace_prefix(test.job.script_filename, test.outputdir)) ret.extend([self.replace_prefix(f, test.outputdir) for f in test.keep_files]) return ret
def test_avg(self): res = evaluate(avg([1, 2, 3, 4])) self.assertEqual(2.5, res) # Check result when passing a generator res = evaluate(avg(range(1, 5))) self.assertEqual(2.5, res) # Check with single element container res = evaluate(avg(range(1, 2))) self.assertEqual(1, res) # Check with empty container self.assertRaises(SanityError, evaluate, avg([]))
def extractsingle(patt, filename, tag=0, conv=None, item=0, encoding='utf-8'): """Extract a single value from the capturing group ``tag`` of a matching regex ``patt`` in the file ``filename``. This function is equivalent to ``extractall(patt, filename, tag, conv)[item]``, except that it raises a ``SanityError`` if ``item`` is out of bounds. :arg patt: as in :func:`extractall`. :arg filename: as in :func:`extractall`. :arg encoding: as in :func:`extractall`. :arg tag: as in :func:`extractall`. :arg conv: as in :func:`extractall`. :arg item: the specific element to extract. :returns: The extracted value. :raises reframe.core.exceptions.SanityError: In case of errors. """ try: # Explicitly evaluate the expression here, so as to force any exception # to be thrown in this context and not during the evaluation of an # expression containing this one. return evaluate(extractall(patt, filename, tag, conv, encoding)[item]) except IndexError: raise SanityError("not enough matches of pattern `%s' in file `%s' " "so as to extract item `%s'" % (patt, filename, item))
def assert_not_found(patt, filename, msg=None, encoding='utf-8'): """Assert that regex pattern ``patt`` is not found in the file ``filename``. This is the inverse of :func:`assert_found()`. :returns: ``True`` on success. :raises reframe.core.exceptions.SanityError: if assertion fails. """ try: evaluate(assert_found(patt, filename, msg, encoding)) except SanityError: return True else: error_msg = msg or "pattern `{0}' found in `{1}'" raise SanityError(_format(error_msg, patt, filename))
def test_extractall(self): # Check numeric groups res = evaluate(extractall('Step: (?P<no>\d+)', self.tempfile, 1)) for expected, v in builtins.enumerate(res, start=1): self.assertEqual(str(expected), v) # Check named groups res = evaluate(extractall('Step: (?P<no>\d+)', self.tempfile, 'no')) for expected, v in builtins.enumerate(res, start=1): self.assertEqual(str(expected), v) # Check convert function res = evaluate( extractall('Step: (?P<no>\d+)', self.tempfile, 'no', builtins.int)) for expected, v in builtins.enumerate(res, start=1): self.assertEqual(expected, v)
def test_chain(self): list1 = ['A', 'B', 'C'] list2 = ['D', 'E', 'F'] chain1 = evaluate(chain(make_deferrable(list1), list2)) chain2 = itertools.chain(list1, list2) self.assertTrue( builtins.all((a == b for a, b in builtins.zip(chain1, chain2))))
def test_map(self): l = [1, 2, 3] dm = map(lambda x: 2 * x + 1, l) for i, x in enumerate(dm, start=1): self.assertEqual(2 * i + 1, x) # Alternative test self.assertEqual([3, 5, 7], list(evaluate(dm)))
def test_filter(self): df = filter(lambda x: x if x % 2 else None, make_deferrable([1, 2, 3, 4, 5])) for i, x in enumerate(df, start=1): self.assertEqual(2 * i - 1, x) # Alternative testing self.assertEqual([1, 3, 5], list(evaluate(df)))
def test_findall(self): res = evaluate(findall('Step: \d+', self.tempfile)) self.assertEqual(3, builtins.len(res)) res = evaluate(findall('Step:.*', self.tempfile)) self.assertEqual(3, builtins.len(res)) res = evaluate(findall('Step: [12]', self.tempfile)) self.assertEqual(2, builtins.len(res)) # Check the matches for expected, match in builtins.zip(['Step: 1', 'Step: 2'], res): self.assertEqual(expected, match.group(0)) # Check groups res = evaluate(findall('Step: (?P<no>\d+)', self.tempfile)) for step, match in builtins.enumerate(res, start=1): self.assertEqual(step, builtins.int(match.group(1))) self.assertEqual(step, builtins.int(match.group('no')))
def check_sanity(self): """The sanity checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the sanity check fails. """ if self.sanity_patterns is None: raise SanityError('sanity_patterns not set') with os_ext.change_dir(self._stagedir): success = evaluate(self.sanity_patterns) if not success: raise SanityError()
def read_timestamps_sorted(self): from reframe.core.deferrable import evaluate self.begin_stamps = [] self.end_stamps = [] for c in self.debug_policy.checks: with open(evaluate(c.stdout), 'r') as f: self.begin_stamps.append(float(f.readline().strip())) self.end_stamps.append(float(f.readline().strip())) self.begin_stamps.sort() self.end_stamps.sort()
def check_performance(self): """The performance checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the performance check fails. """ if self.perf_patterns is None: return with os_ext.change_dir(self._stagedir): # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them # even if the don't meet the reference. for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) if key not in self.reference: raise SanityError( "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) self._perfvalues[key] = (tag, value, *self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) for values in self._perfvalues.values(): tag, val, ref, low_thres, high_thres, *_ = values try: evaluate( assert_reference( val, ref, low_thres, high_thres, msg=('failed to meet reference: %s={0}, ' 'expected {1} (l={2}, u={3})' % tag), )) except SanityError as e: raise PerformanceError(e)
def assert_found(patt, filename, msg=None, encoding='utf-8'): """Assert that regex pattern ``patt`` is found in the file ``filename``. :arg patt: The regex pattern to search. Any standard Python `regular expression <https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`_ is accepted. :arg filename: The name of the file to examine. Any :class:`OSError` raised while processing the file will be propagated as a :class:`reframe.core.exceptions.SanityError`. :arg encoding: The name of the encoding used to decode the file. :returns: ``True`` on success. :raises reframe.core.exceptions.SanityError: if assertion fails. """ num_matches = count(finditer(patt, filename, encoding)) try: evaluate(assert_true(num_matches)) except SanityError: error_msg = msg or "pattern `{0}' not found in `{1}'" raise SanityError(_format(error_msg, patt, filename)) else: return True
def check_performance(self): """The performance checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the performance check fails. """ if self.perf_patterns is None: return with os_ext.change_dir(self._stagedir): for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) try: ref, low_thres, high_thres = self.reference[key] self._perf_logger.info('value: %s, reference: %s' % (value, self.reference[key])) except KeyError: raise SanityError( "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) evaluate(assert_reference(value, ref, low_thres, high_thres))
def do_sanity_check(self): failures = [] devices_found = set( sn.extractall( r'^\s*([^,]*),\s*Detected devices: %s' % self.num_gpus_per_node, self.stdout, 1)) evaluate( sn.assert_eq(self.job.num_tasks, len(devices_found), msg='requested {0} node(s), got {1} (nodelist: %s)' % ','.join(sorted(devices_found)))) good_nodes = set( sn.extractall(r'^\s*([^,]*),\s*NID\s*=\s*\S+\s+Result = PASS', self.stdout, 1)) evaluate( sn.assert_eq(devices_found, good_nodes, msg='check failed on the following node(s): %s' % ','.join(sorted(devices_found - good_nodes)))) # Sanity is fine, fill in the perf. patterns based on the exact node id for nodename in devices_found: for xfer_kind in ('h2d', 'd2h', 'd2d'): for devno in range(self.num_gpus_per_node): perfvar = '%s_gpu_%s_%s_bw' % (nodename, devno, xfer_kind) perfvar = 'bw_%s_%s_gpu_%s' % (xfer_kind, nodename, devno) self.perf_patterns[perfvar] = sn.extractsingle( self._xfer_pattern(xfer_kind, devno, nodename), self.stdout, 1, float, 0) partname = self.current_partition.fullname refkey = '%s:%s' % (partname, perfvar) bwkey = '%s:%s' % (partname, xfer_kind) self.reference[refkey] = self.__bwref[bwkey] return True
def read_timestamps(self, tasks): """Read the timestamps and sort them to permit simple concurrency tests.""" from reframe.core.deferrable import evaluate self.begin_stamps = [] self.end_stamps = [] for t in tasks: with open(evaluate(t.check.stdout), 'r') as f: self.begin_stamps.append(float(f.readline().strip())) self.end_stamps.append(float(f.readline().strip())) self.begin_stamps.sort() self.end_stamps.sort()
def findall(patt, filename, encoding='utf-8'): """Get all matches of regex ``patt`` in ``filename``. :arg patt: The regex pattern to search. Any standard Python `regular expression <https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`_ is accepted. :arg filename: The name of the file to examine. :arg encoding: The name of the encoding used to decode the file. :returns: A list of raw `regex match objects <https://docs.python.org/3.6/library/re.html#match-objects>`_. :raises reframe.core.exceptions.SanityError: In case an :class:`OSError` is raised while processing ``filename``. """ return list(evaluate(x) for x in finditer(patt, filename, encoding))
def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None): """Assert that value ``val`` respects the reference value ``ref``. :arg val: The value to check. :arg ref: The reference value. :arg lower_thres: The lower threshold value expressed as a negative decimal fraction of the reference value. Must be in [-1, 0] for ref >= 0.0 and in [-inf, 0] for ref < 0.0. If ``None``, no lower thresholds is applied. :arg upper_thres: The upper threshold value expressed as a decimal fraction of the reference value. Must be in [0, inf] for ref >= 0.0 and in [0, 1] for ref < 0.0. If ``None``, no upper thresholds is applied. :returns: ``True`` on success. :raises reframe.core.exceptions.SanityError: if assertion fails or if the lower and upper thresholds do not have appropriate values. """ if lower_thres is not None: lower_thres_limit = -1 if ref >= 0 else None try: evaluate(assert_bounded(lower_thres, lower_thres_limit, 0)) except SanityError: raise SanityError('invalid low threshold value: %s' % lower_thres) from None if upper_thres is not None: upper_thres_limit = None if ref >= 0 else 1 try: evaluate(assert_bounded(upper_thres, 0, upper_thres_limit)) except SanityError: raise SanityError('invalid high threshold value: %s' % upper_thres) from None def calc_bound(thres): if thres is None: return None # Inverse threshold if ref < 0 if ref < 0: thres = -thres return ref * (1 + thres) lower = calc_bound(lower_thres) or float('-inf') upper = calc_bound(upper_thres) or float('inf') try: evaluate(assert_bounded(val, lower, upper)) except SanityError: error_msg = '{0} is beyond reference value {1} (l={2}, u={3})' raise SanityError(_format(error_msg, val, ref, lower, upper)) else: return True
def test_extractall_custom_conv(self): res = evaluate( sn.extractall('Step: (\d+)', self.tempfile, 1, lambda x: int(x))) for expected, v in enumerate(res, start=1): self.assertEqual(expected, v) # Check error in custom function self.assertRaises( SanityError, evaluate, sn.extractall('Step: (\d+)', self.tempfile, conv=lambda x: int(x))) # Check error with a callable object class C: def __call__(self, x): return int(x) self.assertRaises( SanityError, evaluate, sn.extractall('Step: (\d+)', self.tempfile, conv=C()))
def extractall(patt, filename, tag=0, conv=None, encoding='utf-8'): """Extract all values from the capturing group ``tag`` of a matching regex ``patt`` in the file ``filename``. :arg patt: The regex pattern to search. Any standard Python `regular expression <https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`_ is accepted. :arg filename: The name of the file to examine. :arg encoding: The name of the encoding used to decode the file. :arg tag: The regex capturing group to be extracted. Group ``0`` refers always to the whole match. Since the file is processed line by line, this means that group ``0`` returns the whole line that was matched. :arg conv: A callable that takes a single argument and returns a new value. If provided, it will be used to convert the extracted values before returning them. :returns: A list of the extracted values from the matched regex. :raises reframe.core.exceptions.SanityError: In case of errors. """ return list( evaluate(x) for x in extractiter(patt, filename, tag, conv, encoding))
def check_performance(self): """The performance checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the performance check fails. """ if self.perf_patterns is None: return with os_ext.change_dir(self._stagedir): # Check if default reference perf values are provided and # store all the variables tested in the performance check has_default = False variables = set() for key, ref in self.reference.items(): keyparts = key.split(self.reference.scope_separator) system = keyparts[0] varname = keyparts[-1] try: unit = ref[3] except IndexError: unit = None variables.add((varname, unit)) if system == '*': has_default = True break if not has_default: if not variables: # If empty, it means that self.reference was empty, so try # to infer their name from perf_patterns variables = {(name, None) for name in self.perf_patterns.keys()} for var in variables: name, unit = var ref_tuple = (0, None, None) if unit: ref_tuple += (unit, ) self.reference.update({'*': {name: ref_tuple}}) # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them # even if the don't meet the reference. for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) if key not in self.reference: raise SanityError( "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) self._perfvalues[key] = (value, *self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) for key, values in self._perfvalues.items(): val, ref, low_thres, high_thres, *_ = values tag = key.split(':')[-1] try: evaluate( assert_reference( val, ref, low_thres, high_thres, msg=('failed to meet reference: %s={0}, ' 'expected {1} (l={2}, u={3})' % tag))) except SanityError as e: raise PerformanceError(e)
def test_ixor(self): v = V(7) dv = make_deferrable(v) dv ^= V(7) evaluate(dv) self.assertEqual(0, v._value)
def test_ior(self): v = V(2) dv = make_deferrable(v) dv |= V(5) evaluate(dv) self.assertEqual(7, v._value)
def test_extractsingle_encoding(self): res = evaluate( extractsingle(r'Odyssey', self.utf16_file, encoding='utf-16')) self.assertNotEqual(-1, res.find('Odyssey'))
def test_extractall_encoding(self): res = evaluate( extractall(r'Odyssey', self.utf16_file, encoding='utf-16')) self.assertEqual(1, len(res))
def test_findall_encoding(self): res = evaluate( sn.findall('Odyssey', self.utf16_file, encoding='utf-16')) self.assertEqual(1, len(res))
def test_sorted(self): l = [2, 3, 1] ds = sorted(l) self.assertEqual([1, 2, 3], list(evaluate(ds)))
def test_setattr(self): dset = setattr(self, '_a', 5) self.assertEqual(0, self._a) evaluate(dset) self.assertEqual(5, self._a)
def test_reversed(self): l = [1, 2, 3] dr = reversed(l) self.assertEqual([3, 2, 1], list(evaluate(dr)))