def test(self):
baseline_dir, result_dir = _image_directories(self._func)
for fignum, baseline in zip(plt.get_fignums(), self._baseline_images):
for extension in self._extensions:
will_fail = not extension in comparable_formats()
if will_fail:
fail_msg = 'Cannot compare %s files on this system' % extension
else:
fail_msg = 'No failure expected'
orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension
if extension == 'eps' and not os.path.exists(orig_expected_fname):
orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf'
expected_fname = make_test_filename(os.path.join(
result_dir, os.path.basename(orig_expected_fname)), 'expected')
actual_fname = os.path.join(result_dir, baseline) + '.' + extension
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
will_fail = True
fail_msg = (
"Do not have baseline image {0} because this "
"file does not exist: {1}".format(
expected_fname,
orig_expected_fname
)
)
@knownfailureif(
will_fail, fail_msg,
known_exception_class=ImageComparisonFailure)
def do_test():
figure = plt.figure(fignum)
if self._remove_text:
self.remove_text(figure)
figure.savefig(actual_fname, **self._savefig_kwarg)
err = compare_images(expected_fname, actual_fname,
self._tol, in_decorator=True)
try:
if not os.path.exists(expected_fname):
raise ImageComparisonFailure(
'image does not exist: %s' % expected_fname)
if err:
raise ImageComparisonFailure(
'images not close: %(actual)s vs. %(expected)s '
'(RMS %(rms).3f)'%err)
except ImageComparisonFailure:
if not check_freetype_version(self._freetype_version):
raise KnownFailureTest(
"Mismatched version of freetype. Test requires '%s', you have '%s'" %
(self._freetype_version, ft2font.__freetype_version__))
raise
yield (do_test,)
def test_image_comparison_expect_rms(im1, im2, tol, expect_rms):
"""
Compare two images, expecting a particular RMS error.
im1 and im2 are filenames relative to the baseline_dir directory.
tol is the tolerance to pass to compare_images.
expect_rms is the expected RMS value, or None. If None, the test will
succeed if compare_images succeeds. Otherwise, the test will succeed if
compare_images fails and returns an RMS error almost equal to this value.
"""
baseline_dir, result_dir = map(Path, _image_directories(lambda: "dummy"))
# Copy both "baseline" and "test" image to result_dir, so that 1)
# compare_images writes the diff to result_dir, rather than to the source
# tree and 2) the baseline image doesn't appear missing to triage_tests.py.
result_im1 = make_test_filename(result_dir / im1, "expected")
shutil.copyfile(baseline_dir / im1, result_im1)
result_im2 = result_dir / im1
shutil.copyfile(baseline_dir / im2, result_im2)
results = compare_images(result_im1,
result_im2,
tol=tol,
in_decorator=True)
if expect_rms is None:
assert results is None
else:
assert results is not None
assert results['rms'] == approx(expect_rms, abs=1e-4)
def test(self):
baseline_dir, result_dir = _image_directories(self._func)
for fignum, baseline in zip(plt.get_fignums(), self._baseline_images):
for extension in self._extensions:
will_fail = not extension in comparable_formats()
if will_fail:
fail_msg = 'Cannot compare %s files on this system' % extension
else:
fail_msg = 'No failure expected'
orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension
if extension == 'eps' and not os.path.exists(orig_expected_fname):
orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf'
expected_fname = make_test_filename(os.path.join(
result_dir, os.path.basename(orig_expected_fname)), 'expected')
actual_fname = os.path.join(result_dir, baseline) + '.' + extension
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
will_fail = True
fail_msg = 'Do not have baseline image %s' % expected_fname
@knownfailureif(
will_fail, fail_msg,
known_exception_class=ImageComparisonFailure)
def do_test():
figure = plt.figure(fignum)
if self._remove_text:
self.remove_text(figure)
figure.savefig(actual_fname, **self._savefig_kwarg)
plt.close(figure)
err = compare_images(expected_fname, actual_fname,
self._tol, in_decorator=True)
try:
if not os.path.exists(expected_fname):
raise ImageComparisonFailure(
'image does not exist: %s' % expected_fname)
if err:
# TODO: uri: Fix this miserable hack:
pass
#raise ImageComparisonFailure(
#'images not close: %(actual)s vs. '
#'%(expected)s (RMS %(rms).3f)' % err)
except ImageComparisonFailure:
if not check_freetype_version(self._freetype_version):
raise KnownFailure(
"Mismatched version of freetype. Test requires '%s', you have '%s'" %
(self._freetype_version, ft2font.__freetype_version__))
raise
yield (do_test,)
def copy_baseline(self, baseline, extension):
baseline_path = os.path.join(self.baseline_dir, baseline)
orig_expected_fname = baseline_path + '.' + extension
if extension == 'eps' and not os.path.exists(orig_expected_fname):
orig_expected_fname = baseline_path + '.pdf'
expected_fname = make_test_filename(os.path.join(
self.result_dir, os.path.basename(orig_expected_fname)), 'expected')
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
xfail("Do not have baseline image {0} because this "
"file does not exist: {1}".format(expected_fname,
orig_expected_fname))
return expected_fname
def _(pth1, pth2, *, tol: int, basename: str = ""):
def fmt_descr(descr):
if basename != "":
return f"{descr} ({basename})"
else:
return descr
pth1, pth2 = Path(pth1), Path(pth2)
try:
result = compare_images(str(pth1), str(pth2), tol=tol)
assert result is None, result
except Exception as e:
diff_pth = make_test_filename(pth2, 'failed-diff')
add_nunit_attachment(str(pth1), fmt_descr("Expected"))
add_nunit_attachment(str(pth2), fmt_descr("Result"))
if Path(diff_pth).is_file():
add_nunit_attachment(str(diff_pth), fmt_descr("Difference"))
raise e
def copy_baseline(self, baseline, extension):
baseline_path = os.path.join(self.baseline_dir, baseline)
orig_expected_fname = baseline_path + '.' + extension
if extension == 'eps' and not os.path.exists(orig_expected_fname):
orig_expected_fname = baseline_path + '.pdf'
expected_fname = make_test_filename(os.path.join(
self.result_dir, os.path.basename(orig_expected_fname)), 'expected')
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
reason = ("Do not have baseline image {0} because this "
"file does not exist: {1}".format(expected_fname,
orig_expected_fname))
if is_called_from_pytest():
import pytest
pytest.xfail(reason)
else:
from ._nose import knownfail
knownfail(reason)
return expected_fname
def copy_baseline(self, baseline, extension):
baseline_path = os.path.join(self.baseline_dir, baseline)
orig_expected_fname = baseline_path + '.' + extension
if extension == 'eps' and not os.path.exists(orig_expected_fname):
orig_expected_fname = baseline_path + '.pdf'
expected_fname = make_test_filename(os.path.join(
self.result_dir, os.path.basename(orig_expected_fname)), 'expected')
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
reason = ("Do not have baseline image {0} because this "
"file does not exist: {1}".format(expected_fname,
orig_expected_fname))
if is_called_from_pytest():
import pytest
pytest.xfail(reason)
else:
from ._nose import knownfail
knownfail(reason)
return expected_fname
def _copy_baseline(self, baseline, extension):
''' Copy baseline image with given extension to result directory. '''
__tracebackhide__ = True # pylint: disable=unused-variable
cls = self.__class__
base_ext = baseline + '.' + extension
# Original baseline file.
baseline_fname = os.path.join(cls.baseline_dir, base_ext)
if extension == 'eps' and not os.path.exists(baseline_fname):
baseline_fname = baseline_fname[:len('eps')] + 'pdf'
# Copied expected file.
expected_fname = mplcmp.make_test_filename(os.path.join(
cls.result_dir, os.path.basename(baseline_fname)), 'expected')
self.assertTrue(os.path.exists(baseline_fname),
'Do not have baseline image {0} '
'because this file does not exist: {1}'
.format(expected_fname, baseline_fname))
shutil.copyfile(baseline_fname, expected_fname)
return expected_fname
def check_graphic(self, tol=_DEFAULT_IMAGE_TOLERANCE):
"""Checks the CRC matches for the current matplotlib.pyplot figure, and closes the figure."""
test_id = self.id()
figure = plt.gcf()
try:
expected_fname = os.path.join(os.path.dirname(__file__),
'reference', 'visual_tests',
test_id + '.png')
if not os.path.isdir(os.path.dirname(expected_fname)):
os.makedirs(os.path.dirname(expected_fname))
#: The path where the images generated by the tests should go.
image_output_directory = os.path.join(os.path.dirname(__file__),
'result_image_comparison')
if not os.access(image_output_directory, os.W_OK):
if not os.access(os.getcwd(), os.W_OK):
raise IOError('Write access to a local disk is required '
'to run image tests. Run the tests from a '
'current working directory you have write '
'access to to avoid this issue.')
else:
image_output_directory = os.path.join(os.getcwd(), 'result_image_comparison')
result_fname = os.path.join(image_output_directory, test_id + '.png')
if not os.path.isdir(os.path.dirname(result_fname)):
# Handle race-condition where the directories are
# created sometime between the check above and the
# creation attempt below.
try:
os.makedirs(os.path.dirname(result_fname))
except OSError as err:
# Don't care about "File exists"
if err.errno != 17:
raise
# Output filename if the test output a file itself (using -o)
output_fname = os.path.join(os.path.dirname(__file__), test_id+'.png')
# If the test created an output file itself then move that to the results folder, otherwise create an output
if os.path.exists(output_fname):
shutil.move(os.path.join(os.path.dirname(__file__), test_id+'.png'), result_fname)
else:
figure.savefig(result_fname)
if not os.path.exists(expected_fname):
logging.warn('Created image for test %s' % test_id)
shutil.copy2(result_fname, expected_fname)
try:
err = mcompare.compare_images(expected_fname, result_fname, tol=tol)
except ValueError:
failed_name = mcompare.make_test_filename(result_fname, 'failed-diff')
shutil.copy2(os.path.join(os.path.dirname(__file__),
'reference', 'kitten.png'), failed_name)
err = "Images differ in size and so are not comparable"
if _DISPLAY_FIGURES:
if err:
print(('Image comparison would have failed. Message: %s' % err))
plt.show()
else:
assert not err, 'Image comparison failed. Message: %s' % err
finally:
plt.close()
def test(self):
baseline_dir, result_dir = _image_directories(self._func)
if self._style != 'classic':
skip('temporarily disabled until 2.0 tag')
for fignum, baseline in zip(plt.get_fignums(), self._baseline_images):
for extension in self._extensions:
will_fail = not extension in comparable_formats()
if will_fail:
fail_msg = 'Cannot compare %s files on this system' % extension
else:
fail_msg = 'No failure expected'
orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension
if extension == 'eps' and not os.path.exists(orig_expected_fname):
orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf'
expected_fname = make_test_filename(os.path.join(
result_dir, os.path.basename(orig_expected_fname)), 'expected')
actual_fname = os.path.join(result_dir, baseline) + '.' + extension
if os.path.exists(orig_expected_fname):
shutil.copyfile(orig_expected_fname, expected_fname)
else:
will_fail = True
fail_msg = (
"Do not have baseline image {0} because this "
"file does not exist: {1}".format(
expected_fname,
orig_expected_fname
)
)
@knownfailureif(
will_fail, fail_msg,
known_exception_class=ImageComparisonFailure)
def do_test(fignum, actual_fname, expected_fname):
figure = plt.figure(fignum)
if self._remove_text:
self.remove_text(figure)
figure.savefig(actual_fname, **self._savefig_kwarg)
err = compare_images(expected_fname, actual_fname,
self._tol, in_decorator=True)
try:
if not os.path.exists(expected_fname):
raise ImageComparisonFailure(
'image does not exist: %s' % expected_fname)
if err:
raise ImageComparisonFailure(
'images not close: %(actual)s vs. %(expected)s '
'(RMS %(rms).3f)'%err)
except ImageComparisonFailure:
if not check_freetype_version(self._freetype_version):
xfail(
"Mismatched version of freetype. Test requires '%s', you have '%s'" %
(self._freetype_version, ft2font.__freetype_version__))
raise
yield do_test, fignum, actual_fname, expected_fname