def test(self): baseline_dir, result_dir = _image_directories(self._func) for fignum, baseline in zip(plt.get_fignums(), self._baseline_images): for extension in self._extensions: will_fail = not extension in comparable_formats() if will_fail: fail_msg = 'Cannot compare %s files on this system' % extension else: fail_msg = 'No failure expected' orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension if extension == 'eps' and not os.path.exists(orig_expected_fname): orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf' expected_fname = make_test_filename(os.path.join( result_dir, os.path.basename(orig_expected_fname)), 'expected') actual_fname = os.path.join(result_dir, baseline) + '.' + extension if os.path.exists(orig_expected_fname): shutil.copyfile(orig_expected_fname, expected_fname) else: will_fail = True fail_msg = ( "Do not have baseline image {0} because this " "file does not exist: {1}".format( expected_fname, orig_expected_fname ) ) @knownfailureif( will_fail, fail_msg, known_exception_class=ImageComparisonFailure) def do_test(): figure = plt.figure(fignum) if self._remove_text: self.remove_text(figure) figure.savefig(actual_fname, **self._savefig_kwarg) err = compare_images(expected_fname, actual_fname, self._tol, in_decorator=True) try: if not os.path.exists(expected_fname): raise ImageComparisonFailure( 'image does not exist: %s' % expected_fname) if err: raise ImageComparisonFailure( 'images not close: %(actual)s vs. %(expected)s ' '(RMS %(rms).3f)'%err) except ImageComparisonFailure: if not check_freetype_version(self._freetype_version): raise KnownFailureTest( "Mismatched version of freetype. Test requires '%s', you have '%s'" % (self._freetype_version, ft2font.__freetype_version__)) raise yield (do_test,)
def test_image_comparison_expect_rms(im1, im2, tol, expect_rms): """ Compare two images, expecting a particular RMS error. im1 and im2 are filenames relative to the baseline_dir directory. tol is the tolerance to pass to compare_images. expect_rms is the expected RMS value, or None. If None, the test will succeed if compare_images succeeds. Otherwise, the test will succeed if compare_images fails and returns an RMS error almost equal to this value. """ baseline_dir, result_dir = map(Path, _image_directories(lambda: "dummy")) # Copy both "baseline" and "test" image to result_dir, so that 1) # compare_images writes the diff to result_dir, rather than to the source # tree and 2) the baseline image doesn't appear missing to triage_tests.py. result_im1 = make_test_filename(result_dir / im1, "expected") shutil.copyfile(baseline_dir / im1, result_im1) result_im2 = result_dir / im1 shutil.copyfile(baseline_dir / im2, result_im2) results = compare_images(result_im1, result_im2, tol=tol, in_decorator=True) if expect_rms is None: assert results is None else: assert results is not None assert results['rms'] == approx(expect_rms, abs=1e-4)
def test(self): baseline_dir, result_dir = _image_directories(self._func) for fignum, baseline in zip(plt.get_fignums(), self._baseline_images): for extension in self._extensions: will_fail = not extension in comparable_formats() if will_fail: fail_msg = 'Cannot compare %s files on this system' % extension else: fail_msg = 'No failure expected' orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension if extension == 'eps' and not os.path.exists(orig_expected_fname): orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf' expected_fname = make_test_filename(os.path.join( result_dir, os.path.basename(orig_expected_fname)), 'expected') actual_fname = os.path.join(result_dir, baseline) + '.' + extension if os.path.exists(orig_expected_fname): shutil.copyfile(orig_expected_fname, expected_fname) else: will_fail = True fail_msg = 'Do not have baseline image %s' % expected_fname @knownfailureif( will_fail, fail_msg, known_exception_class=ImageComparisonFailure) def do_test(): figure = plt.figure(fignum) if self._remove_text: self.remove_text(figure) figure.savefig(actual_fname, **self._savefig_kwarg) plt.close(figure) err = compare_images(expected_fname, actual_fname, self._tol, in_decorator=True) try: if not os.path.exists(expected_fname): raise ImageComparisonFailure( 'image does not exist: %s' % expected_fname) if err: # TODO: uri: Fix this miserable hack: pass #raise ImageComparisonFailure( #'images not close: %(actual)s vs. ' #'%(expected)s (RMS %(rms).3f)' % err) except ImageComparisonFailure: if not check_freetype_version(self._freetype_version): raise KnownFailure( "Mismatched version of freetype. Test requires '%s', you have '%s'" % (self._freetype_version, ft2font.__freetype_version__)) raise yield (do_test,)
def copy_baseline(self, baseline, extension): baseline_path = os.path.join(self.baseline_dir, baseline) orig_expected_fname = baseline_path + '.' + extension if extension == 'eps' and not os.path.exists(orig_expected_fname): orig_expected_fname = baseline_path + '.pdf' expected_fname = make_test_filename(os.path.join( self.result_dir, os.path.basename(orig_expected_fname)), 'expected') if os.path.exists(orig_expected_fname): shutil.copyfile(orig_expected_fname, expected_fname) else: xfail("Do not have baseline image {0} because this " "file does not exist: {1}".format(expected_fname, orig_expected_fname)) return expected_fname
def _(pth1, pth2, *, tol: int, basename: str = ""): def fmt_descr(descr): if basename != "": return f"{descr} ({basename})" else: return descr pth1, pth2 = Path(pth1), Path(pth2) try: result = compare_images(str(pth1), str(pth2), tol=tol) assert result is None, result except Exception as e: diff_pth = make_test_filename(pth2, 'failed-diff') add_nunit_attachment(str(pth1), fmt_descr("Expected")) add_nunit_attachment(str(pth2), fmt_descr("Result")) if Path(diff_pth).is_file(): add_nunit_attachment(str(diff_pth), fmt_descr("Difference")) raise e
def copy_baseline(self, baseline, extension): baseline_path = os.path.join(self.baseline_dir, baseline) orig_expected_fname = baseline_path + '.' + extension if extension == 'eps' and not os.path.exists(orig_expected_fname): orig_expected_fname = baseline_path + '.pdf' expected_fname = make_test_filename(os.path.join( self.result_dir, os.path.basename(orig_expected_fname)), 'expected') if os.path.exists(orig_expected_fname): shutil.copyfile(orig_expected_fname, expected_fname) else: reason = ("Do not have baseline image {0} because this " "file does not exist: {1}".format(expected_fname, orig_expected_fname)) if is_called_from_pytest(): import pytest pytest.xfail(reason) else: from ._nose import knownfail knownfail(reason) return expected_fname
def _copy_baseline(self, baseline, extension): ''' Copy baseline image with given extension to result directory. ''' __tracebackhide__ = True # pylint: disable=unused-variable cls = self.__class__ base_ext = baseline + '.' + extension # Original baseline file. baseline_fname = os.path.join(cls.baseline_dir, base_ext) if extension == 'eps' and not os.path.exists(baseline_fname): baseline_fname = baseline_fname[:len('eps')] + 'pdf' # Copied expected file. expected_fname = mplcmp.make_test_filename(os.path.join( cls.result_dir, os.path.basename(baseline_fname)), 'expected') self.assertTrue(os.path.exists(baseline_fname), 'Do not have baseline image {0} ' 'because this file does not exist: {1}' .format(expected_fname, baseline_fname)) shutil.copyfile(baseline_fname, expected_fname) return expected_fname
def check_graphic(self, tol=_DEFAULT_IMAGE_TOLERANCE): """Checks the CRC matches for the current matplotlib.pyplot figure, and closes the figure.""" test_id = self.id() figure = plt.gcf() try: expected_fname = os.path.join(os.path.dirname(__file__), 'reference', 'visual_tests', test_id + '.png') if not os.path.isdir(os.path.dirname(expected_fname)): os.makedirs(os.path.dirname(expected_fname)) #: The path where the images generated by the tests should go. image_output_directory = os.path.join(os.path.dirname(__file__), 'result_image_comparison') if not os.access(image_output_directory, os.W_OK): if not os.access(os.getcwd(), os.W_OK): raise IOError('Write access to a local disk is required ' 'to run image tests. Run the tests from a ' 'current working directory you have write ' 'access to to avoid this issue.') else: image_output_directory = os.path.join(os.getcwd(), 'result_image_comparison') result_fname = os.path.join(image_output_directory, test_id + '.png') if not os.path.isdir(os.path.dirname(result_fname)): # Handle race-condition where the directories are # created sometime between the check above and the # creation attempt below. try: os.makedirs(os.path.dirname(result_fname)) except OSError as err: # Don't care about "File exists" if err.errno != 17: raise # Output filename if the test output a file itself (using -o) output_fname = os.path.join(os.path.dirname(__file__), test_id+'.png') # If the test created an output file itself then move that to the results folder, otherwise create an output if os.path.exists(output_fname): shutil.move(os.path.join(os.path.dirname(__file__), test_id+'.png'), result_fname) else: figure.savefig(result_fname) if not os.path.exists(expected_fname): logging.warn('Created image for test %s' % test_id) shutil.copy2(result_fname, expected_fname) try: err = mcompare.compare_images(expected_fname, result_fname, tol=tol) except ValueError: failed_name = mcompare.make_test_filename(result_fname, 'failed-diff') shutil.copy2(os.path.join(os.path.dirname(__file__), 'reference', 'kitten.png'), failed_name) err = "Images differ in size and so are not comparable" if _DISPLAY_FIGURES: if err: print(('Image comparison would have failed. Message: %s' % err)) plt.show() else: assert not err, 'Image comparison failed. Message: %s' % err finally: plt.close()
def test(self): baseline_dir, result_dir = _image_directories(self._func) if self._style != 'classic': skip('temporarily disabled until 2.0 tag') for fignum, baseline in zip(plt.get_fignums(), self._baseline_images): for extension in self._extensions: will_fail = not extension in comparable_formats() if will_fail: fail_msg = 'Cannot compare %s files on this system' % extension else: fail_msg = 'No failure expected' orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension if extension == 'eps' and not os.path.exists(orig_expected_fname): orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf' expected_fname = make_test_filename(os.path.join( result_dir, os.path.basename(orig_expected_fname)), 'expected') actual_fname = os.path.join(result_dir, baseline) + '.' + extension if os.path.exists(orig_expected_fname): shutil.copyfile(orig_expected_fname, expected_fname) else: will_fail = True fail_msg = ( "Do not have baseline image {0} because this " "file does not exist: {1}".format( expected_fname, orig_expected_fname ) ) @knownfailureif( will_fail, fail_msg, known_exception_class=ImageComparisonFailure) def do_test(fignum, actual_fname, expected_fname): figure = plt.figure(fignum) if self._remove_text: self.remove_text(figure) figure.savefig(actual_fname, **self._savefig_kwarg) err = compare_images(expected_fname, actual_fname, self._tol, in_decorator=True) try: if not os.path.exists(expected_fname): raise ImageComparisonFailure( 'image does not exist: %s' % expected_fname) if err: raise ImageComparisonFailure( 'images not close: %(actual)s vs. %(expected)s ' '(RMS %(rms).3f)'%err) except ImageComparisonFailure: if not check_freetype_version(self._freetype_version): xfail( "Mismatched version of freetype. Test requires '%s', you have '%s'" % (self._freetype_version, ft2font.__freetype_version__)) raise yield do_test, fignum, actual_fname, expected_fname