def _load_config_pairs(self, configs):
"""Loads the results of all tests, across all builders (based on the
files within self._actuals_root), compares them across two configs,
and stores the summary in self._results.
Args:
configs: tuple of strings; pair of configs to compare
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
actual_builder_dicts = self._read_builder_dicts_from_root(
self._actuals_root)
configA, configB = configs
logging.info('Comparing configs %s and %s...' % (configA, configB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=configs, diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=configs, diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
builders = sorted(actual_builder_dicts.keys())
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info(
'Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
actual_results_for_this_builder = (
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[
result_type]
if not results_of_this_type:
continue
tests_found = set()
for image_name in sorted(results_of_this_type.keys()):
(test,
_) = results.IMAGE_FILENAME_RE.match(image_name).groups()
tests_found.add(test)
for test in tests_found:
# Get image_relative_url (or None) for each of configA, configB
image_name_A = results.IMAGE_FILENAME_FORMATTER % (test,
configA)
configA_image_relative_url = ConfigComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type.get(
image_name_A),
test_name=test)
image_name_B = results.IMAGE_FILENAME_FORMATTER % (test,
configB)
configB_image_relative_url = ConfigComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type.get(
image_name_B),
test_name=test)
# If we have images for at least one of these two configs,
# add them to our list.
if configA_image_relative_url or configB_image_relative_url:
if configA_image_relative_url == configB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif not configA_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not configB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__RESULT_TYPE:
result_type,
results.KEY__EXTRACOLUMNS__BUILDER:
builder,
results.KEY__EXTRACOLUMNS__TEST:
test,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include a 'config' column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMNS__CONFIG:
'TODO',
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageA_relative_url=configA_image_relative_url,
imageB_relative_url=configB_image_relative_url,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for test '
'"%s", builder "%s"' % (test, builder))
# pylint: disable=W0201
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(),
}
def _load_actual_and_expected(self):
"""Loads the results of all tests, across all builders (based on the
files within self._actuals_root and self._expected_root),
and stores them in self._results.
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
actual_builder_dicts = self._read_builder_dicts_from_root(
self._actuals_root)
logging.info('Reading expected-results JSON files from %s...' %
self._expected_root)
expected_builder_dicts = self._read_builder_dicts_from_root(
self._expected_root)
all_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__FAILUREIGNORED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__FAILUREIGNORED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
# Only consider builders we have both expected and actual results for.
# Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
# (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
# builder')
actual_builder_set = set(actual_builder_dicts.keys())
expected_builder_set = set(expected_builder_dicts.keys())
builders = sorted(actual_builder_set.intersection(expected_builder_set))
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
actual_results_for_this_builder = (
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[result_type]
if not results_of_this_type:
continue
for image_name in sorted(results_of_this_type.keys()):
(test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
actual_image_relative_url = (
ExpectationComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type[image_name],
test_name=test))
# Default empty expectations; overwrite these if we find any real ones
expectations_per_test = None
expected_image_relative_url = None
expectations_dict = None
try:
expectations_per_test = (
expected_builder_dicts
[builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name])
# TODO(epoger): assumes a single allowed digest per test, which is
# fine; see https://code.google.com/p/skia/issues/detail?id=1787
expected_image_hashtype_and_digest = (
expectations_per_test
[gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
expected_image_relative_url = (
ExpectationComparisons._create_relative_url(
hashtype_and_digest=expected_image_hashtype_and_digest,
test_name=test))
expectations_dict = {}
for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
expectations_dict[field] = expectations_per_test.get(field)
except (KeyError, TypeError):
# There are several cases in which we would expect to find
# no expectations for a given test:
#
# 1. result_type == NOCOMPARISON
# There are no expectations for this test yet!
#
# 2. alternate rendering mode failures (e.g. serialized)
# In cases like
# https://code.google.com/p/skia/issues/detail?id=1684
# ('tileimagefilter GM test failing in serialized render mode'),
# the gm-actuals will list a failure for the alternate
# rendering mode even though we don't have explicit expectations
# for the test (the implicit expectation is that it must
# render the same in all rendering modes).
#
# Don't log type 1, because it is common.
# Log other types, because they are rare and we should know about
# them, but don't throw an exception, because we need to keep our
# tools working in the meanwhile!
if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
logging.warning('No expectations found for test: %s' % {
results.KEY__EXTRACOLUMNS__BUILDER: builder,
results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
'image_name': image_name,
})
# If this test was recently rebaselined, it will remain in
# the 'failed' set of actuals until all the bots have
# cycled (although the expectations have indeed been set
# from the most recent actuals). Treat these as successes
# instead of failures.
#
# TODO(epoger): Do we need to do something similar in
# other cases, such as when we have recently marked a test
# as ignoreFailure but it still shows up in the 'failed'
# category? Maybe we should not rely on the result_type
# categories recorded within the gm_actuals AT ALL, and
# instead evaluate the result_type ourselves based on what
# we see in expectations vs actual checksum?
if expected_image_relative_url == actual_image_relative_url:
updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
(test in self._ignore_failures_on_these_tests)):
updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
else:
updated_result_type = result_type
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type,
results.KEY__EXTRACOLUMNS__BUILDER: builder,
results.KEY__EXTRACOLUMNS__TEST: test,
results.KEY__EXTRACOLUMNS__CONFIG: config,
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageA_relative_url=expected_image_relative_url,
imageB_relative_url=actual_image_relative_url,
expectations=expectations_dict,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except Exception:
logging.exception('got exception while creating new ImagePair')
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
}
def test_endToEnd(self):
"""Tests ImagePair, using a real ImageDiffDB to download real images.
TODO(epoger): Either in addition to or instead of this end-to-end test,
we should perform some tests using either:
1. a mock ImageDiffDB, or
2. a real ImageDiffDB that doesn't hit Google Storage looking for input
image files (maybe a file:// IMG_URL_BASE)
"""
# params for each self-test:
#
# inputs:
# 0. imageA_relative_URL
# 1. imageB_relative_URL
# 2. expectations dict
# 3. extra_columns dict
# expected output:
# 4. expected result of ImagePair.as_dict()
selftests = [
[
# inputs:
'arcofzorro/16206093933823793653.png',
'arcofzorro/16206093933823793653.png',
None,
{
'builder': 'MyBuilder',
'test': 'MyTest',
},
# expected output:
{
'extraColumns': {
'builder': 'MyBuilder',
'test': 'MyTest',
},
'imageAUrl': 'arcofzorro/16206093933823793653.png',
'imageBUrl': 'arcofzorro/16206093933823793653.png',
'isDifferent': False,
},
],
[
# inputs:
'arcofzorro/16206093933823793653.png',
'arcofzorro/13786535001616823825.png',
None,
None,
# expected output:
{
'differenceData': {
'maxDiffPerChannel': [255, 255, 247],
'numDifferingPixels':
662,
'percentDifferingPixels':
0.0662,
'perceptualDifference':
0.06620300000000157,
'diffUrl':
'arcofzorro_16206093933823793653_png_png-vs-' +
'arcofzorro_13786535001616823825_png_png.png',
'whiteDiffUrl':
'arcofzorro_16206093933823793653_png_png' +
'-vs-arcofzorro_13786535001616823825_png_png.png',
},
'imageAUrl': 'arcofzorro/16206093933823793653.png',
'imageBUrl': 'arcofzorro/13786535001616823825.png',
'isDifferent': True,
},
],
[
# inputs:
'gradients_degenerate_2pt/10552995703607727960.png',
'gradients_degenerate_2pt/11198253335583713230.png',
{
'ignoreFailure': True,
'bugs': [1001, 1002],
},
{
'builder': 'MyBuilder',
'test': 'MyTest',
},
# expected output:
{
'differenceData': {
'maxDiffPerChannel': [255, 0, 255],
'numDifferingPixels':
102400,
'percentDifferingPixels':
100.00,
'perceptualDifference':
100.00,
'diffUrl':
'gradients_degenerate_2pt_10552995703607727960' +
'_png_png-vs-gradients_degenerate_2pt_' +
'11198253335583713230_png_png.png',
'whiteDiffUrl':
'gradients_degenerate_2pt_' +
'10552995703607727960_png_png-vs-' +
'gradients_degenerate_2pt_11198253335583713230' +
'_png_png.png'
},
'expectations': {
'bugs': [1001, 1002],
'ignoreFailure': True,
},
'extraColumns': {
'builder': 'MyBuilder',
'test': 'MyTest',
},
'imageAUrl':
'gradients_degenerate_2pt/10552995703607727960.png',
'imageBUrl':
'gradients_degenerate_2pt/11198253335583713230.png',
'isDifferent': True,
},
],
# Test fix for http://skbug.com/2368 -- how do we handle an ImagePair
# missing one of its images?
[
# inputs:
'arcofzorro/16206093933823793653.png',
'nonexistentDir/111111.png',
{
'ignoreFailure': True,
'bugs': [1001, 1002],
},
{
'builder': 'MyBuilder',
'test': 'MyTest',
},
# expected output:
{
'expectations': {
'bugs': [1001, 1002],
'ignoreFailure': True,
},
'extraColumns': {
'builder': 'MyBuilder',
'test': 'MyTest',
},
'imageAUrl': 'arcofzorro/16206093933823793653.png',
'imageBUrl': 'nonexistentDir/111111.png',
'isDifferent': True,
},
],
# One of the two images is missing, but download_all_images=True so we
# should download it anyway.
[
# inputs:
None,
'arcofzorro/13786535001616823825.png',
None,
None,
# expected output:
{
'imageAUrl': None,
'imageBUrl': 'arcofzorro/13786535001616823825.png',
'isDifferent': True,
},
],
]
db = imagediffdb.ImageDiffDB(self.temp_dir)
for selftest in selftests:
image_pair = imagepair.ImagePair(image_diff_db=db,
imageA_base_url=IMG_URL_BASE,
imageB_base_url=IMG_URL_BASE,
imageA_relative_url=selftest[0],
imageB_relative_url=selftest[1],
expectations=selftest[2],
extra_columns=selftest[3],
download_all_images=True)
self.assertEqual(image_pair.as_dict(), selftest[4])
def _load_result_pairs(self, actuals_root, subdirs):
"""Loads all JSON files found within two subdirs in actuals_root,
compares across those two subdirs, and stores the summary in self._results.
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
"""
logging.info(
'Reading actual-results JSON files from %s subdirs within %s...' %
(subdirs, actuals_root))
subdirA, subdirB = subdirs
subdirA_builder_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirA))
subdirB_builder_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirB))
logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs, diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs, diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
builders = sorted(
set(subdirA_builder_dicts.keys() + subdirB_builder_dicts.keys()))
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info(
'Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
# TODO(epoger): This will fail if we have results for this builder in
# subdirA but not subdirB (or vice versa).
subdirA_results = results.BaseComparisons.combine_subdicts(
subdirA_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
subdirB_results = results.BaseComparisons.combine_subdicts(
subdirB_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
image_names = sorted(
set(subdirA_results.keys() + subdirB_results.keys()))
for image_name in image_names:
# The image name may contain funny characters or be ridiculously long
# (see https://code.google.com/p/skia/issues/detail?id=2344#c10 ),
# so make sure we sanitize it before using it in a URL path.
#
# TODO(epoger): Rather than sanitizing/truncating the image name here,
# do it in render_pictures instead.
# Reason: we will need to be consistent in applying this rule, so that
# the process which uploads the files to GS using these paths will
# match the paths created by downstream processes.
# So, we should make render_pictures write out images to paths that are
# "ready to upload" to Google Storage, like gm does.
sanitized_test_name = DISALLOWED_FILEPATH_CHAR_REGEX.sub(
'_', image_name)[:30]
subdirA_image_relative_url = (
results.BaseComparisons._create_relative_url(
hashtype_and_digest=subdirA_results.get(image_name),
test_name=sanitized_test_name))
subdirB_image_relative_url = (
results.BaseComparisons._create_relative_url(
hashtype_and_digest=subdirB_results.get(image_name),
test_name=sanitized_test_name))
# If we have images for at least one of these two subdirs,
# add them to our list.
if subdirA_image_relative_url or subdirB_image_relative_url:
if subdirA_image_relative_url == subdirB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif not subdirA_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not subdirB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMN__RESULT_TYPE:
result_type,
results.KEY__EXTRACOLUMN__BUILDER:
builder,
results.KEY__EXTRACOLUMN__TEST:
image_name,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include a 'config' column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMN__CONFIG:
'TODO',
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=self._image_base_url,
imageA_relative_url=subdirA_image_relative_url,
imageB_relative_url=subdirB_image_relative_url,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for image_name '
'"%s", builder "%s"' % (image_name, builder))
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(),
}
def _create_image_pair(self, image_dict_A, image_dict_B, image_A_base_url,
image_B_base_url, builder_A, render_mode_A,
builder_B, render_mode_B, source_json_file,
source_skp_name, tilenum):
"""Creates an ImagePair object for this pair of images.
Args:
image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image
image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image
image_A_base_url: base URL for image A
image_B_base_url: base URL for image B
builder_A: builder that created image set A or None if unknow
render_mode_A: render mode used to generate image set A or None if
unknown.
builder_B: builder that created image set A or None if unknow
render_mode_B: render mode used to generate image set A or None if
unknown.
source_json_file: string; relative path of the JSON file where this
result came from, within setA and setB.
source_skp_name: string; name of the source SKP file
tilenum: which tile, or None if a wholeimage
Returns:
An ImagePair object, or None if both image_dict_A and image_dict_B are
None.
"""
if (not image_dict_A) and (not image_dict_B):
return None
def _checksum_and_relative_url(dic):
if dic:
return ((dic[gm_json.JSONKEY_IMAGE_CHECKSUMALGORITHM],
int(dic[gm_json.JSONKEY_IMAGE_CHECKSUMVALUE])),
dic[gm_json.JSONKEY_IMAGE_FILEPATH])
else:
return None, None
imageA_checksum, imageA_relative_url = _checksum_and_relative_url(
image_dict_A)
imageB_checksum, imageB_relative_url = _checksum_and_relative_url(
image_dict_B)
if not imageA_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not imageB_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif imageA_checksum == imageB_checksum:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
COLUMN__RESULT_TYPE: result_type,
COLUMN__SOURCE_SKP: source_skp_name,
COLUMN__BUILDER_A: builder_A,
COLUMN__RENDER_MODE_A: render_mode_A,
COLUMN__BUILDER_B: builder_B,
COLUMN__RENDER_MODE_B: render_mode_B,
}
if tilenum == None:
extra_columns_dict[
COLUMN__TILED_OR_WHOLE] = COLUMN__TILED_OR_WHOLE__WHOLE
extra_columns_dict[COLUMN__TILENUM] = 'N/A'
else:
extra_columns_dict[
COLUMN__TILED_OR_WHOLE] = COLUMN__TILED_OR_WHOLE__TILED
extra_columns_dict[COLUMN__TILENUM] = str(tilenum)
try:
return imagepair.ImagePair(
image_diff_db=self._image_diff_db,
imageA_base_url=image_A_base_url,
imageB_base_url=image_B_base_url,
imageA_relative_url=imageA_relative_url,
imageB_relative_url=imageB_relative_url,
extra_columns=extra_columns_dict,
source_json_file=source_json_file,
download_all_images=self._download_all_images)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for'
' urlPair=("%s","%s"), source_skp_name="%s", tilenum="%s"' %
(imageA_relative_url, imageB_relative_url, source_skp_name,
tilenum))
return None
def _create_image_pair(self, test, config, image_dict_A, image_dict_B):
"""Creates an ImagePair object for this pair of images.
Args:
test: string; name of the test
config: string; name of the config
image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image
image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image
Returns:
An ImagePair object, or None if both image_dict_A and image_dict_B are
None.
"""
if (not image_dict_A) and (not image_dict_B):
return None
def _checksum_and_relative_url(dic):
if dic:
return ((dic[gm_json.JSONKEY_IMAGE_CHECKSUMALGORITHM],
dic[gm_json.JSONKEY_IMAGE_CHECKSUMVALUE]),
dic[gm_json.JSONKEY_IMAGE_FILEPATH])
else:
return None, None
imageA_checksum, imageA_relative_url = _checksum_and_relative_url(
image_dict_A)
imageB_checksum, imageB_relative_url = _checksum_and_relative_url(
image_dict_B)
if not imageA_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not imageB_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif imageA_checksum == imageB_checksum:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__CONFIG: config,
results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
results.KEY__EXTRACOLUMNS__TEST: test,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include this column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMNS__BUILDER: 'TODO',
}
try:
return imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=self._image_base_url,
imageA_relative_url=imageA_relative_url,
imageB_relative_url=imageB_relative_url,
extra_columns=extra_columns_dict)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for'
' test="%s", config="%s", urlPair=("%s","%s")' % (
test, config, imageA_relative_url, imageB_relative_url))
return None
def _create_image_pair(self, image_dict_A, image_dict_B, source_skp_name,
tilenum):
"""Creates an ImagePair object for this pair of images.
Args:
image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image
image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image
source_skp_name: string; name of the source SKP file
tilenum: which tile, or None if a wholeimage
Returns:
An ImagePair object, or None if both image_dict_A and image_dict_B are
None.
"""
if (not image_dict_A) and (not image_dict_B):
return None
def _checksum_and_relative_url(dic):
if dic:
return ((dic[gm_json.JSONKEY_IMAGE_CHECKSUMALGORITHM],
dic[gm_json.JSONKEY_IMAGE_CHECKSUMVALUE]),
dic[gm_json.JSONKEY_IMAGE_FILEPATH])
else:
return None, None
imageA_checksum, imageA_relative_url = _checksum_and_relative_url(
image_dict_A)
imageB_checksum, imageB_relative_url = _checksum_and_relative_url(
image_dict_B)
if not imageA_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not imageB_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif imageA_checksum == imageB_checksum:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
COLUMN__RESULT_TYPE: result_type,
COLUMN__SOURCE_SKP: source_skp_name,
}
if tilenum == None:
extra_columns_dict[COLUMN__TILED_OR_WHOLE] = 'whole'
extra_columns_dict[COLUMN__TILENUM] = 'N/A'
else:
extra_columns_dict[COLUMN__TILED_OR_WHOLE] = 'tiled'
extra_columns_dict[COLUMN__TILENUM] = str(tilenum)
try:
return imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=self._image_base_gs_url,
imageA_relative_url=imageA_relative_url,
imageB_relative_url=imageB_relative_url,
extra_columns=extra_columns_dict,
download_all_images=self._download_all_images)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for'
' urlPair=("%s","%s"), source_skp_name="%s", tilenum="%s"' %
(imageA_relative_url, imageB_relative_url, source_skp_name,
tilenum))
return None
def test_endToEnd(self):
"""Tests ImagePair, using a real ImageDiffDB to download real images.
TODO(epoger): Either in addition to or instead of this end-to-end test,
we should perform some tests using either:
1. a mock ImageDiffDB, or
2. a real ImageDiffDB that doesn't hit Google Storage looking for input
image files (maybe a file:// IMG_URL_BASE)
"""
# params for each self-test:
#
# inputs:
# 0. imageA_relative_URL
# 1. imageB_relative_URL
# 2. expectations dict
# 3. extra_columns dict
# expected output:
# 4. expected result of ImagePair.as_dict()
selftests = [
[
# inputs:
'arcofzorro/16206093933823793653.png',
'arcofzorro/16206093933823793653.png',
None,
{
'builder': 'MyBuilder',
'test': 'MyTest',
},
# expected output:
{
'extraColumns': {
'builder': 'MyBuilder',
'test': 'MyTest',
},
'imageAUrl': 'arcofzorro/16206093933823793653.png',
'imageBUrl': 'arcofzorro/16206093933823793653.png',
'isDifferent': False,
},
],
[
# inputs:
'arcofzorro/16206093933823793653.png',
'arcofzorro/13786535001616823825.png',
None,
None,
# expected output:
{
'differenceData': {
'maxDiffPerChannel': [255, 255, 247],
'numDifferingPixels': 662,
'percentDifferingPixels': 0.0662,
'perceptualDifference': 0.06620000000000914,
'weightedDiffMeasure': 0.01127756555171088,
},
'imageAUrl': 'arcofzorro/16206093933823793653.png',
'imageBUrl': 'arcofzorro/13786535001616823825.png',
'isDifferent': True,
},
],
[
# inputs:
'gradients_degenerate_2pt/10552995703607727960.png',
'gradients_degenerate_2pt/11198253335583713230.png',
{
'ignoreFailure': True,
'bugs': [1001, 1002],
},
{
'builder': 'MyBuilder',
'test': 'MyTest',
},
# expected output:
{
'differenceData': {
'maxDiffPerChannel': [255, 0, 255],
'numDifferingPixels': 102400,
'percentDifferingPixels': 100.00,
'perceptualDifference': 100.00,
'weightedDiffMeasure': 66.66666666666667,
},
'expectations': {
'bugs': [1001, 1002],
'ignoreFailure': True,
},
'extraColumns': {
'builder': 'MyBuilder',
'test': 'MyTest',
},
'imageAUrl':
'gradients_degenerate_2pt/10552995703607727960.png',
'imageBUrl':
'gradients_degenerate_2pt/11198253335583713230.png',
'isDifferent': True,
},
],
]
db = imagediffdb.ImageDiffDB(self._temp_dir)
for selftest in selftests:
image_pair = imagepair.ImagePair(image_diff_db=db,
base_url=IMG_URL_BASE,
imageA_relative_url=selftest[0],
imageB_relative_url=selftest[1],
expectations=selftest[2],
extra_columns=selftest[3])
self.assertEqual(image_pair.as_dict(), selftest[4])