def _compare_image(self, driver_output, expected_driver_outputs):
failures = []
# If we didn't produce a hash file, this test must be text-only.
if driver_output.image_hash is None:
return failures
if not expected_driver_outputs.image:
failures.append(test_failures.FailureMissingImage())
elif not expected_driver_outputs.image_hash:
failures.append(test_failures.FailureMissingImageHash())
elif driver_output.image_hash != expected_driver_outputs.image_hash:
failures.append(test_failures.FailureImageHashMismatch())
return failures
def _failures_from_fail_row(self, row):
# Look at all anchors in this row, and guess what type
# of new-run-webkit-test failures they equate to.
failures = set()
test_name = None
for anchor in row.findAll("a"):
anchor_text = unicode(anchor.string)
if not test_name:
test_name = anchor_text
continue
if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text:
failures.add(test_failures.FailureImageHashMismatch())
elif anchor_text in ["expected", "actual", "diff", "pretty diff"]:
failures.add(test_failures.FailureTextMismatch())
else:
log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text)
# FIXME: Its possible the row contained no links due to ORWT brokeness.
# We should probably assume some type of failure anyway.
return failures
self.write_output_files(port, filename, '.checksum',
test_args.hash, expected_hash,
encoding="ascii",
print_text_diffs=False)
self._copy_output_png(filename, test_args.png_path, '-actual.png')
self._copy_output_png(filename, expected_png_file, '-expected.png')
# Even though we only use the result in one codepath below but we
# still need to call CreateImageDiff for other codepaths.
images_are_different = self._create_image_diff(port, filename, configuration)
if expected_hash == '':
failures.append(test_failures.FailureMissingImageHash(self))
elif test_args.hash != expected_hash:
if images_are_different:
failures.append(test_failures.FailureImageHashMismatch(self))
else:
failures.append(test_failures.FailureImageHashIncorrect(self))
return failures
def diff_files(self, port, file1, file2):
"""Diff two image files.
Args:
file1, file2: full paths of the files to compare.
Returns:
True if two files are different.
False otherwise.
"""
def compare_output(self, port, filename, output, test_args, configuration):
"""Implementation of CompareOutput that checks the output image and
checksum against the expected files from the LayoutTest directory.
"""
failures = []
# If we didn't produce a hash file, this test must be text-only.
if test_args.hash is None:
return failures
# If we're generating a new baseline, we pass.
if test_args.new_baseline or test_args.reset_results:
self._save_baseline_files(filename, test_args.png_path,
test_args.hash, test_args.new_baseline)
return failures
# Compare hashes.
expected_hash_file = self._port.expected_filename(
filename, '.checksum')
expected_png_file = self._port.expected_filename(filename, '.png')
# FIXME: We repeat this pattern often, we should share code.
expected_hash = ''
if os.path.exists(expected_hash_file):
with codecs.open(expected_hash_file, "r", "ascii") as file:
expected_hash = file.read()
if not os.path.isfile(expected_png_file):
# Report a missing expected PNG file.
self.write_output_files(port,
filename,
'.checksum',
test_args.hash,
expected_hash,
encoding="ascii",
print_text_diffs=False)
self._copy_output_png(filename, test_args.png_path, '-actual.png')
failures.append(test_failures.FailureMissingImage())
return failures
elif test_args.hash == expected_hash:
# Hash matched (no diff needed, okay to return).
return failures
self.write_output_files(port,
filename,
'.checksum',
test_args.hash,
expected_hash,
encoding="ascii",
print_text_diffs=False)
self._copy_output_png(filename, test_args.png_path, '-actual.png')
self._copy_output_png(filename, expected_png_file, '-expected.png')
# Even though we only use the result in one codepath below but we
# still need to call CreateImageDiff for other codepaths.
images_are_different = self._create_image_diff(port, filename,
configuration)
if expected_hash == '':
failures.append(test_failures.FailureMissingImageHash())
elif test_args.hash != expected_hash:
if images_are_different:
failures.append(test_failures.FailureImageHashMismatch())
else:
failures.append(test_failures.FailureImageHashIncorrect())
return failures