def analyze(self): rewrite = (self.global_env['options'].rewrite and not self.expect_failure) failures = [] # Check for the test output itself diff = fileutils.diff(self.expected_file, self.output_file, ignore_white_chars=False) if diff: if rewrite: new_baseline = self.read_file(self.output_file) with open(self.original_expected_file, 'w') as f: f.write(new_baseline) self.result.actual_output += diff failures.append('output is not as expected{}'.format( ' (baseline updated)' if rewrite else '')) # Check memory issues if asked to if self.valgrind_errors: self.result.actual_output += ( 'Valgrind reported the following errors:\n{}'.format( self.valgrind.format_report(self.valgrind_errors))) failures.append('memory issues detected') if failures: self.set_failure(' | '.join(failures)) else: self.set_passed()
def analyze(self): rewrite = (self.global_env['options'].rewrite and not self.expect_failure) failures = [] # Check for the test output itself diff = fileutils.diff(self.expected_file, self.output_file, ignore_white_chars=False) if diff: if rewrite: new_baseline = self.read_file(self.output_file) with open(self.original_expected_file, 'w') as f: f.write(new_baseline) self.result.actual_output += diff failures.append('output is not as expected{}'.format( ' (baseline updated)' if rewrite else '' )) # Check memory issues if asked to if self.valgrind_errors: self.result.actual_output += ( 'Valgrind reported the following errors:\n{}'.format( self.valgrind.format_report(self.valgrind_errors) ) ) failures.append('memory isuses detected') if failures: self.set_failure(' | '.join(failures)) else: self.set_passed()
def analyze(self): diff = fileutils.diff(self.test_working_dir(self.expected_file), self.test_working_dir(self.out_file)) rewrite = (self.global_env['options'].rewrite and not self.expect_failure) # Determine the status of this test, ignoring expected failure (for # now). if diff: if rewrite: new_baseline = self.read_file( self.test_working_dir(self.out_file) ) self.write_file( self.test_orig_dir(self.expected_file), new_baseline ) self.result.actual_output += diff failed = True message = 'diff in output' else: failed = False message = '' self.set_result_status(failed, message)
def diff(self, expected_out): """Return self.out followed by a diff self.cmd_out and expected_out. PARAMETERS expected_out: A string containing the expected output. """ diff_str = diff(expected_out.splitlines(), self.cmd_out.splitlines()) return '%s\n\nDiff:\n\n%s' % (self.image, diff_str)
def assertOutputEqual(self, expected, actual): """same as assertEqual but for strings and with a diff if not equal. This a convenience function that allows us to quickly diagnose what's wrong when the output does not match... """ self.assertEqual( expected, actual, "Diff:\n\n%s" % diff(expected.splitlines(), actual.splitlines()))
def analyze(self): # Check for the test output itself diff = fileutils.diff(self.expected_file, self.output_file, ignore_white_chars=False) if diff: self.set_failure('output is not as expected') self.result.actual_output += diff else: self.set_passed()
def analyze(self): diff = fileutils.diff(self.test_working_dir(self.expected_file), self.test_working_dir(self.out_file)) # Determine the status of this test, ignoring expected failure (for # now). if diff: self.result.actual_output += diff failed = True message = 'diff in output' else: failed = False message = '' self.set_result_status(failed, message)
def analyze_diff(self, expected=None, actual=None, strip_cr=True, replace_backslashes=True): """Set status based on diff analysis. If there is no difference test status is set to PASSED, otherwise it is set to FAILED. diff string is stored in self.result.diff. :param expected: if None then self.result.expected_output is taken otherwise parameter expected is used :type expected: str | None :param actual: if None then self.result.actual_output is taken otherwise parameter actual is used :type expected: str | None :param strip_cr: if True, strip cr from both expected and actual :type strip_cr: bool :param replace_backslashes: if True, replace backslashes by slashes in expected and actual :type replace_backslashes: bool """ if expected is None: expected = self.result.expected_output if actual is None: actual = self.result.actual_output if strip_cr: actual = actual.replace('\r', '') expected = expected.replace('\r', '') for d in self.subst: logging.debug('%s -> %s' % (d[0], d[1])) expected = re.sub(d[0], d[1], expected) actual = re.sub(d[0], d[1], actual) if replace_backslashes: actual = actual.replace('\\', '/') expected = expected.replace('\\', '/') self.result.diff = diff(expected.splitlines(), actual.splitlines()) if self.result.diff: self.result.set_status('FAILED', 'output diff') else: self.result.set_status('PASSED')
def analyze(self): # RA22-015: For the transition to the concrete syntax, we want to # check-in and test unparsing results. super(PythonDriver, self).analyze() rewrite = (self.global_env['options'].rewrite and not self.expect_failure) failures = [] expected_lkt = self.working_dir('expected_concrete_syntax.lkt') actual_lkt = self.working_dir('concrete_syntax.lkt') if not os.path.exists(actual_lkt): return # We just open the file in append mode, to create it if it doesn't # exist. with open(expected_lkt, 'a+'): pass # Check for the test output itself diff = fileutils.diff(expected_lkt, actual_lkt, ignore_white_chars=False) if diff: if rewrite: new_baseline = self.read_file(actual_lkt) with open( os.path.join(self.test_dir, 'expected_concrete_syntax.lkt'), 'w') as f: f.write(new_baseline) self.result.actual_output += diff failures.append('output is not as expected{}'.format( ' (baseline updated)' if rewrite else '')) if failures: self.set_failure(' | '.join(failures)) else: self.set_passed()
def analyze(self): rewrite = (self.global_env['options'].rewrite and not self.expect_failure) failures = [] # Check for the test output itself diff = fileutils.diff(self.expected_file, self.output_file, ignore_white_chars=False) if diff: if rewrite: new_baseline = self.read_file(self.output_file) with open(self.original_expected_file, 'w') as f: f.write(new_baseline) self.result.actual_output += diff failures.append('output is not as expected{}'.format( ' (baseline updated)' if rewrite else '')) if failures: self.set_failure(' | '.join(failures)) else: self.set_passed()
def analyze(self): rewrite = (self.global_env['options'].rewrite and not self.expect_failure) failures = [] # Check for the test output itself diff = fileutils.diff(self.expected_file, self.output_file, ignore_white_chars=False) if diff: if rewrite: new_baseline = self.read_file(self.output_file) with open(self.original_expected_file, 'w') as f: f.write(new_baseline) self.result.actual_output += diff failures.append('output is not as expected{}'.format( ' (baseline updated)' if rewrite else '' )) if failures: self.set_failure(' | '.join(failures)) else: self.set_passed()
def diff(first_file, second_file): return fileutils.diff(first_file, second_file, ignore_white_chars=False)
def analyze(self, ignore_white_chars=True): """Compute test status. :param ignore_white_chars: in the default driver difference in white chars are ignored. This parameter allow the user to change that behavior. In that case the user should override the analyze method in its own driver and call this method with ignore_white_chars set to False. :type ignore_white_chars: bool This method should set the final value of 'result' attribute """ # Retrieve the outputs and see if we match some of the CRASH or DEAD # patterns output = split_file(self.output, ignore_errors=True) if output: tmp = "\n".join(output) for pattern in self.get_status_filter(): if re.search(pattern[0], tmp): self.result.update(pattern[1]) break # If the test status has not been updated compare output with the # baseline if self.result['result'] == 'UNKNOWN': # Retrieve expected output expected = split_file(self.opt_results['OUT'], ignore_errors=True) # Process output and expected output with registered filters expected = self.apply_output_filter(expected) output = self.apply_output_filter(output) # Save the filtered output (might be needed by some developpers to # create more easily baselines). echo_to_file(self.output_filtered, output) d = diff(expected, output, ignore_white_chars=ignore_white_chars) if d: logging.debug(d) self.result['result'] = 'DIFF' if len(expected) == 0: self.result['msg'] = 'unexpected output' else: self.result['msg'] = 'output' diff_file = open(self.diff_output, 'w') diff_file.write(d) diff_file.close() else: self.result = {'result': 'OK', 'msg': '', 'is_failure': False} self.result['is_failure'] = IS_STATUS_FAILURE[self.result['result']] # self.opt_results['XFAIL'] contains the XFAIL comment or False # The status should be set to XFAIL even if the comment is empty if not isinstance(self.opt_results['XFAIL'], bool) or \ self.opt_results['XFAIL']: if self.result['result'] in ['DIFF', 'CRASH']: self.result.update({ 'result': 'XFAIL', 'msg': self.opt_results['XFAIL'] }) elif self.result['result'] == 'OK': self.result.update({ 'result': 'UOK', 'msg': self.opt_results['XFAIL'] })