def test_success(self):
"""Assembles some ImagePairs into an ImagePairSet, and validates results.
"""
image_pairs = [
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_1_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_2_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_3_AS_DICT),
]
expected_imageset_dict = {
'extraColumnHeaders': {
'builder': {
'headerText': 'builder',
'isFilterable': True,
'isSortable': True,
'valuesAndCounts': {
'MyBuilder': 3
},
},
'test': {
'headerText': 'which GM test',
'headerUrl': 'http://learn/about/gm/tests',
'isFilterable': True,
'isSortable': False,
},
},
'imagePairs': [
IMAGEPAIR_1_AS_DICT,
IMAGEPAIR_2_AS_DICT,
IMAGEPAIR_3_AS_DICT,
],
'imageSets': [
{
'baseUrl': BASE_URL_1,
'description': SET_A_DESCRIPTION,
},
{
'baseUrl': BASE_URL_1,
'description': SET_B_DESCRIPTION,
},
],
}
image_pair_set = imagepairset.ImagePairSet(
descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION))
for image_pair in image_pairs:
image_pair_set.add_image_pair(image_pair)
# The 'builder' column header uses the default settings,
# but the 'test' column header has manual adjustments.
image_pair_set.set_column_header_factory(
'test',
column.ColumnHeaderFactory(
header_text='which GM test',
header_url='http://learn/about/gm/tests',
is_filterable=True,
is_sortable=False,
include_values_and_counts=False))
self.assertEqual(image_pair_set.as_dict(), expected_imageset_dict)
def test_mismatched_base_url(self):
"""Confirms that mismatched base_urls will cause an exception."""
image_pair_set = imagepairset.ImagePairSet(
diff_base_url=DIFF_BASE_URL)
image_pair_set.add_image_pair(
MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_1_AS_DICT))
image_pair_set.add_image_pair(
MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_2_AS_DICT))
with self.assertRaises(Exception):
image_pair_set.add_image_pair(
MockImagePair(base_url=BASE_URL_2,
dict_to_return=IMAGEPAIR_3_AS_DICT))
def test_missing_column_ids(self):
"""Confirms that passing truncated column_ids_in_order to as_dict()
will cause an exception."""
image_pair_set = imagepairset.ImagePairSet(diff_base_url=DIFF_BASE_URL)
image_pair_set.add_image_pair(
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_1_AS_DICT))
image_pair_set.add_image_pair(
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_2_AS_DICT))
# Call as_dict() with default or reasonable column_ids_in_order.
image_pair_set.as_dict()
image_pair_set.as_dict(column_ids_in_order=['test', 'builder'])
image_pair_set.as_dict(
column_ids_in_order=['test', 'builder', 'extra'])
# Call as_dict() with not enough column_ids.
with self.assertRaises(Exception):
image_pair_set.as_dict(column_ids_in_order=['builder'])
def _load_config_pairs(self, configs):
"""Loads the results of all tests, across all builders (based on the
files within self._actuals_root), compares them across two configs,
and stores the summary in self._results.
Args:
configs: tuple of strings; pair of configs to compare
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
actual_builder_dicts = self._read_builder_dicts_from_root(
self._actuals_root)
configA, configB = configs
logging.info('Comparing configs %s and %s...' % (configA, configB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=configs, diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=configs, diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
builders = sorted(actual_builder_dicts.keys())
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info(
'Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
actual_results_for_this_builder = (
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[
result_type]
if not results_of_this_type:
continue
tests_found = set()
for image_name in sorted(results_of_this_type.keys()):
(test,
_) = results.IMAGE_FILENAME_RE.match(image_name).groups()
tests_found.add(test)
for test in tests_found:
# Get image_relative_url (or None) for each of configA, configB
image_name_A = results.IMAGE_FILENAME_FORMATTER % (test,
configA)
configA_image_relative_url = ConfigComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type.get(
image_name_A),
test_name=test)
image_name_B = results.IMAGE_FILENAME_FORMATTER % (test,
configB)
configB_image_relative_url = ConfigComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type.get(
image_name_B),
test_name=test)
# If we have images for at least one of these two configs,
# add them to our list.
if configA_image_relative_url or configB_image_relative_url:
if configA_image_relative_url == configB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif not configA_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not configB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__RESULT_TYPE:
result_type,
results.KEY__EXTRACOLUMNS__BUILDER:
builder,
results.KEY__EXTRACOLUMNS__TEST:
test,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include a 'config' column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMNS__CONFIG:
'TODO',
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageA_relative_url=configA_image_relative_url,
imageB_relative_url=configB_image_relative_url,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for test '
'"%s", builder "%s"' % (test, builder))
# pylint: disable=W0201
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(),
}
def test_success(self):
"""Assembles some ImagePairs into an ImagePairSet, and validates results.
"""
image_pairs = [
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_1_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_2_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_3_AS_DICT),
]
expected_imageset_dict = {
'extraColumnHeaders': {
'builder': {
'headerText': 'builder',
'isFilterable': True,
'isSortable': True,
'useFreeformFilter': False,
'valuesAndCounts': [('MyBuilder', 3)],
},
'test': {
'headerText': 'which GM test',
'headerUrl': 'http://learn/about/gm/tests',
'isFilterable': True,
'isSortable': False,
'useFreeformFilter': False,
'valuesAndCounts': [('test1', 1), ('test2', 1),
('test3', 1)],
},
},
'extraColumnOrder': ['builder', 'test'],
'imagePairs': [
IMAGEPAIR_1_AS_DICT,
IMAGEPAIR_2_AS_DICT,
IMAGEPAIR_3_AS_DICT,
],
'imageSets': {
'imageA': {
'baseUrl': BASE_URL_1,
'description': SET_A_DESCRIPTION,
},
'imageB': {
'baseUrl': BASE_URL_1,
'description': SET_B_DESCRIPTION,
},
'diffs': {
'baseUrl': DIFF_BASE_URL + '/diffs',
'description': 'color difference per channel',
},
'whiteDiffs': {
'baseUrl': DIFF_BASE_URL + '/whitediffs',
'description': 'differing pixels in white',
},
},
}
image_pair_set = imagepairset.ImagePairSet(
descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION),
diff_base_url=DIFF_BASE_URL)
for image_pair in image_pairs:
image_pair_set.add_image_pair(image_pair)
# The 'builder' column header uses the default settings,
# but the 'test' column header has manual adjustments.
image_pair_set.set_column_header_factory(
'test',
column.ColumnHeaderFactory(
header_text='which GM test',
header_url='http://learn/about/gm/tests',
is_filterable=True,
is_sortable=False))
self.assertEqual(image_pair_set.as_dict(), expected_imageset_dict)
def _load_actual_and_expected(self):
"""Loads the results of all tests, across all builders (based on the
files within self._actuals_root and self._expected_root),
and stores them in self._results.
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
actual_builder_dicts = self._read_builder_dicts_from_root(
self._actuals_root)
logging.info('Reading expected-results JSON files from %s...' %
self._expected_root)
expected_builder_dicts = self._read_builder_dicts_from_root(
self._expected_root)
all_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__FAILUREIGNORED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__FAILUREIGNORED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
# Only consider builders we have both expected and actual results for.
# Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
# (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
# builder')
actual_builder_set = set(actual_builder_dicts.keys())
expected_builder_set = set(expected_builder_dicts.keys())
builders = sorted(actual_builder_set.intersection(expected_builder_set))
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
actual_results_for_this_builder = (
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[result_type]
if not results_of_this_type:
continue
for image_name in sorted(results_of_this_type.keys()):
(test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
actual_image_relative_url = (
ExpectationComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type[image_name],
test_name=test))
# Default empty expectations; overwrite these if we find any real ones
expectations_per_test = None
expected_image_relative_url = None
expectations_dict = None
try:
expectations_per_test = (
expected_builder_dicts
[builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name])
# TODO(epoger): assumes a single allowed digest per test, which is
# fine; see https://code.google.com/p/skia/issues/detail?id=1787
expected_image_hashtype_and_digest = (
expectations_per_test
[gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
expected_image_relative_url = (
ExpectationComparisons._create_relative_url(
hashtype_and_digest=expected_image_hashtype_and_digest,
test_name=test))
expectations_dict = {}
for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
expectations_dict[field] = expectations_per_test.get(field)
except (KeyError, TypeError):
# There are several cases in which we would expect to find
# no expectations for a given test:
#
# 1. result_type == NOCOMPARISON
# There are no expectations for this test yet!
#
# 2. alternate rendering mode failures (e.g. serialized)
# In cases like
# https://code.google.com/p/skia/issues/detail?id=1684
# ('tileimagefilter GM test failing in serialized render mode'),
# the gm-actuals will list a failure for the alternate
# rendering mode even though we don't have explicit expectations
# for the test (the implicit expectation is that it must
# render the same in all rendering modes).
#
# Don't log type 1, because it is common.
# Log other types, because they are rare and we should know about
# them, but don't throw an exception, because we need to keep our
# tools working in the meanwhile!
if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
logging.warning('No expectations found for test: %s' % {
results.KEY__EXTRACOLUMNS__BUILDER: builder,
results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
'image_name': image_name,
})
# If this test was recently rebaselined, it will remain in
# the 'failed' set of actuals until all the bots have
# cycled (although the expectations have indeed been set
# from the most recent actuals). Treat these as successes
# instead of failures.
#
# TODO(epoger): Do we need to do something similar in
# other cases, such as when we have recently marked a test
# as ignoreFailure but it still shows up in the 'failed'
# category? Maybe we should not rely on the result_type
# categories recorded within the gm_actuals AT ALL, and
# instead evaluate the result_type ourselves based on what
# we see in expectations vs actual checksum?
if expected_image_relative_url == actual_image_relative_url:
updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
(test in self._ignore_failures_on_these_tests)):
updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
else:
updated_result_type = result_type
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type,
results.KEY__EXTRACOLUMNS__BUILDER: builder,
results.KEY__EXTRACOLUMNS__TEST: test,
results.KEY__EXTRACOLUMNS__CONFIG: config,
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageA_relative_url=expected_image_relative_url,
imageB_relative_url=actual_image_relative_url,
expectations=expectations_dict,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except Exception:
logging.exception('got exception while creating new ImagePair')
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
}
def _load_result_pairs(self, actuals_root, subdirs):
"""Loads all JSON files found within two subdirs in actuals_root,
compares across those two subdirs, and stores the summary in self._results.
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
"""
logging.info(
'Reading actual-results JSON files from %s subdirs within %s...' %
(subdirs, actuals_root))
subdirA, subdirB = subdirs
subdirA_builder_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirA))
subdirB_builder_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirB))
logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs, diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs, diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
builders = sorted(
set(subdirA_builder_dicts.keys() + subdirB_builder_dicts.keys()))
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info(
'Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
# TODO(epoger): This will fail if we have results for this builder in
# subdirA but not subdirB (or vice versa).
subdirA_results = results.BaseComparisons.combine_subdicts(
subdirA_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
subdirB_results = results.BaseComparisons.combine_subdicts(
subdirB_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
image_names = sorted(
set(subdirA_results.keys() + subdirB_results.keys()))
for image_name in image_names:
# The image name may contain funny characters or be ridiculously long
# (see https://code.google.com/p/skia/issues/detail?id=2344#c10 ),
# so make sure we sanitize it before using it in a URL path.
#
# TODO(epoger): Rather than sanitizing/truncating the image name here,
# do it in render_pictures instead.
# Reason: we will need to be consistent in applying this rule, so that
# the process which uploads the files to GS using these paths will
# match the paths created by downstream processes.
# So, we should make render_pictures write out images to paths that are
# "ready to upload" to Google Storage, like gm does.
sanitized_test_name = DISALLOWED_FILEPATH_CHAR_REGEX.sub(
'_', image_name)[:30]
subdirA_image_relative_url = (
results.BaseComparisons._create_relative_url(
hashtype_and_digest=subdirA_results.get(image_name),
test_name=sanitized_test_name))
subdirB_image_relative_url = (
results.BaseComparisons._create_relative_url(
hashtype_and_digest=subdirB_results.get(image_name),
test_name=sanitized_test_name))
# If we have images for at least one of these two subdirs,
# add them to our list.
if subdirA_image_relative_url or subdirB_image_relative_url:
if subdirA_image_relative_url == subdirB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif not subdirA_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not subdirB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMN__RESULT_TYPE:
result_type,
results.KEY__EXTRACOLUMN__BUILDER:
builder,
results.KEY__EXTRACOLUMN__TEST:
image_name,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include a 'config' column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMN__CONFIG:
'TODO',
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=self._image_base_url,
imageA_relative_url=subdirA_image_relative_url,
imageB_relative_url=subdirB_image_relative_url,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for image_name '
'"%s", builder "%s"' % (image_name, builder))
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(),
}
def _load_result_pairs(self, setA_root, setB_root, setA_section,
setB_section):
"""Loads all JSON image summaries from 2 directory trees and compares them.
TODO(stephana): This method is only called from within __init__(); it might
make more sense to just roll the content of this method into __init__().
Args:
setA_root: root directory containing JSON summaries of rendering results
setB_root: root directory containing JSON summaries of rendering results
setA_section: which section (gm_json.JSONKEY_ACTUALRESULTS or
gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setA
setB_section: which section (gm_json.JSONKEY_ACTUALRESULTS or
gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setB
Returns the summary of all image diff results (or None, depending on
self._prefetch_only).
"""
logging.info('Reading JSON image summaries from dirs %s and %s...' %
(setA_root, setB_root))
setA_dicts = self.read_dicts_from_root(setA_root)
setB_dicts = self.read_dicts_from_root(setB_root)
logging.info('Comparing summary dicts...')
all_image_pairs = imagepairset.ImagePairSet(
descriptions=(self._setA_label, self._setB_label),
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=(self._setA_label, self._setB_label),
diff_base_url=self._diff_base_url)
# Override settings for columns that should be filtered using freeform text.
for column_id in FREEFORM_COLUMN_IDS:
factory = column.ColumnHeaderFactory(header_text=column_id,
use_freeform_filter=True)
all_image_pairs.set_column_header_factory(
column_id=column_id, column_header_factory=factory)
failing_image_pairs.set_column_header_factory(
column_id=column_id, column_header_factory=factory)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=COLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=COLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
logging.info('Starting to add imagepairs to queue.')
self._image_diff_db.log_queue_size_if_changed(limit_verbosity=False)
union_dict_paths = sorted(set(setA_dicts.keys() + setB_dicts.keys()))
num_union_dict_paths = len(union_dict_paths)
dict_num = 0
for dict_path in union_dict_paths:
dict_num += 1
logging.info(
'Asynchronously requesting pixel diffs for dict #%d of %d, "%s"...'
% (dict_num, num_union_dict_paths, dict_path))
dictA = self.get_default(setA_dicts, None, dict_path)
self._validate_dict_version(dictA)
dictA_results = self.get_default(dictA, {}, setA_section)
dictB = self.get_default(setB_dicts, None, dict_path)
self._validate_dict_version(dictB)
dictB_results = self.get_default(dictB, {}, setB_section)
image_A_base_url = self.get_default(
setA_dicts, self._image_base_gs_url, dict_path,
gm_json.JSONKEY_IMAGE_BASE_GS_URL)
image_B_base_url = self.get_default(
setB_dicts, self._image_base_gs_url, dict_path,
gm_json.JSONKEY_IMAGE_BASE_GS_URL)
# get the builders and render modes for each set
builder_A = self.get_default(dictA, None,
gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_BUILDER)
render_mode_A = self.get_default(
dictA, None, gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_RENDER_MODE)
builder_B = self.get_default(dictB, None,
gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_BUILDER)
render_mode_B = self.get_default(
dictB, None, gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_RENDER_MODE)
skp_names = sorted(set(dictA_results.keys() +
dictB_results.keys()))
# Just for manual testing... truncate to an arbitrary subset.
if self.truncate_results:
skp_names = skp_names[1:3]
for skp_name in skp_names:
imagepairs_for_this_skp = []
whole_image_A = self.get_default(
dictA_results, None, skp_name,
gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
whole_image_B = self.get_default(
dictB_results, None, skp_name,
gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
imagepairs_for_this_skp.append(
self._create_image_pair(image_dict_A=whole_image_A,
image_dict_B=whole_image_B,
image_A_base_url=image_A_base_url,
image_B_base_url=image_B_base_url,
builder_A=builder_A,
render_mode_A=render_mode_A,
builder_B=builder_B,
render_mode_B=render_mode_B,
source_json_file=dict_path,
source_skp_name=skp_name,
tilenum=None))
tiled_images_A = self.get_default(
dictA_results, [], skp_name,
gm_json.JSONKEY_SOURCE_TILEDIMAGES)
tiled_images_B = self.get_default(
dictB_results, [], skp_name,
gm_json.JSONKEY_SOURCE_TILEDIMAGES)
if tiled_images_A or tiled_images_B:
num_tiles_A = len(tiled_images_A)
num_tiles_B = len(tiled_images_B)
num_tiles = max(num_tiles_A, num_tiles_B)
for tile_num in range(num_tiles):
imagepairs_for_this_skp.append(
self._create_image_pair(
image_dict_A=(tiled_images_A[tile_num]
if tile_num < num_tiles_A else
None),
image_dict_B=(tiled_images_B[tile_num]
if tile_num < num_tiles_B else
None),
image_A_base_url=image_A_base_url,
image_B_base_url=image_B_base_url,
builder_A=builder_A,
render_mode_A=render_mode_A,
builder_B=builder_B,
render_mode_B=render_mode_B,
source_json_file=dict_path,
source_skp_name=skp_name,
tilenum=tile_num))
for one_imagepair in imagepairs_for_this_skp:
if one_imagepair:
all_image_pairs.add_image_pair(one_imagepair)
result_type = one_imagepair.extra_columns_dict\
[COLUMN__RESULT_TYPE]
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(one_imagepair)
logging.info('Finished adding imagepairs to queue.')
self._image_diff_db.log_queue_size_if_changed(limit_verbosity=False)
if self._prefetch_only:
return None
else:
return {
results.KEY__HEADER__RESULTS_ALL:
all_image_pairs.as_dict(
column_ids_in_order=ORDERED_COLUMN_IDS),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(
column_ids_in_order=ORDERED_COLUMN_IDS),
}
def _load_result_pairs(self, actuals_root, subdirs):
"""Loads all JSON files found within two subdirs in actuals_root,
compares across those two subdirs, and stores the summary in self._results.
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
"""
logging.info(
'Reading actual-results JSON files from %s subdirs within %s...' % (
subdirs, actuals_root))
subdirA, subdirB = subdirs
subdirA_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirA))
subdirB_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirB))
logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs,
diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
common_dict_paths = sorted(set(subdirA_dicts.keys() + subdirB_dicts.keys()))
num_common_dict_paths = len(common_dict_paths)
dict_num = 0
for dict_path in common_dict_paths:
dict_num += 1
logging.info('Generating pixel diffs for dict #%d of %d, "%s"...' %
(dict_num, num_common_dict_paths, dict_path))
dictA = subdirA_dicts[dict_path]
dictB = subdirB_dicts[dict_path]
self._validate_dict_version(dictA)
self._validate_dict_version(dictB)
dictA_results = dictA[gm_json.JSONKEY_ACTUALRESULTS]
dictB_results = dictB[gm_json.JSONKEY_ACTUALRESULTS]
skp_names = sorted(set(dictA_results.keys() + dictB_results.keys()))
for skp_name in skp_names:
imagepairs_for_this_skp = []
whole_image_A = RenderedPicturesComparisons.get_multilevel(
dictA_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
whole_image_B = RenderedPicturesComparisons.get_multilevel(
dictB_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
imagepairs_for_this_skp.append(self._create_image_pair(
test=skp_name, config=gm_json.JSONKEY_SOURCE_WHOLEIMAGE,
image_dict_A=whole_image_A, image_dict_B=whole_image_B))
tiled_images_A = RenderedPicturesComparisons.get_multilevel(
dictA_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES)
tiled_images_B = RenderedPicturesComparisons.get_multilevel(
dictB_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES)
# TODO(epoger): Report an error if we find tiles for A but not B?
if tiled_images_A and tiled_images_B:
# TODO(epoger): Report an error if we find a different number of tiles
# for A and B?
num_tiles = len(tiled_images_A)
for tile_num in range(num_tiles):
imagepairs_for_this_skp.append(self._create_image_pair(
test=skp_name,
config='%s-%d' % (gm_json.JSONKEY_SOURCE_TILEDIMAGES, tile_num),
image_dict_A=tiled_images_A[tile_num],
image_dict_B=tiled_images_B[tile_num]))
for imagepair in imagepairs_for_this_skp:
if imagepair:
all_image_pairs.add_image_pair(imagepair)
result_type = imagepair.extra_columns_dict\
[results.KEY__EXTRACOLUMNS__RESULT_TYPE]
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(imagepair)
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
}
