def test_success(self):
"""Assembles some ImagePairs into an ImagePairSet, and validates results.
"""
image_pairs = [
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_1_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_2_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_3_AS_DICT),
]
expected_imageset_dict = {
'extraColumnHeaders': {
'builder': {
'headerText': 'builder',
'isFilterable': True,
'isSortable': True,
'valuesAndCounts': {
'MyBuilder': 3
},
},
'test': {
'headerText': 'which GM test',
'headerUrl': 'http://learn/about/gm/tests',
'isFilterable': True,
'isSortable': False,
},
},
'imagePairs': [
IMAGEPAIR_1_AS_DICT,
IMAGEPAIR_2_AS_DICT,
IMAGEPAIR_3_AS_DICT,
],
'imageSets': [
{
'baseUrl': BASE_URL_1,
'description': SET_A_DESCRIPTION,
},
{
'baseUrl': BASE_URL_1,
'description': SET_B_DESCRIPTION,
},
],
}
image_pair_set = imagepairset.ImagePairSet(
descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION))
for image_pair in image_pairs:
image_pair_set.add_image_pair(image_pair)
# The 'builder' column header uses the default settings,
# but the 'test' column header has manual adjustments.
image_pair_set.set_column_header_factory(
'test',
column.ColumnHeaderFactory(
header_text='which GM test',
header_url='http://learn/about/gm/tests',
is_filterable=True,
is_sortable=False,
include_values_and_counts=False))
self.assertEqual(image_pair_set.as_dict(), expected_imageset_dict)
def get_column_header_factory(self, column_id):
"""Returns the ColumnHeaderFactory object for a particular extraColumn.
Args:
column_id: string; unique ID of this column (must match a key within
an ImagePair's extra_columns dictionary)
"""
column_header_factory = self._column_header_factories.get(column_id, None)
if not column_header_factory:
column_header_factory = column.ColumnHeaderFactory(header_text=column_id)
self._column_header_factories[column_id] = column_header_factory
return column_header_factory
def test_success(self):
"""Assembles some ImagePairs into an ImagePairSet, and validates results.
"""
image_pairs = [
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_1_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_2_AS_DICT),
MockImagePair(base_url=BASE_URL_1,
dict_to_return=IMAGEPAIR_3_AS_DICT),
]
expected_imageset_dict = {
'extraColumnHeaders': {
'builder': {
'headerText': 'builder',
'isFilterable': True,
'isSortable': True,
'useFreeformFilter': False,
'valuesAndCounts': [('MyBuilder', 3)],
},
'test': {
'headerText': 'which GM test',
'headerUrl': 'http://learn/about/gm/tests',
'isFilterable': True,
'isSortable': False,
'useFreeformFilter': False,
'valuesAndCounts': [('test1', 1), ('test2', 1),
('test3', 1)],
},
},
'extraColumnOrder': ['builder', 'test'],
'imagePairs': [
IMAGEPAIR_1_AS_DICT,
IMAGEPAIR_2_AS_DICT,
IMAGEPAIR_3_AS_DICT,
],
'imageSets': {
'imageA': {
'baseUrl': BASE_URL_1,
'description': SET_A_DESCRIPTION,
},
'imageB': {
'baseUrl': BASE_URL_1,
'description': SET_B_DESCRIPTION,
},
'diffs': {
'baseUrl': DIFF_BASE_URL + '/diffs',
'description': 'color difference per channel',
},
'whiteDiffs': {
'baseUrl': DIFF_BASE_URL + '/whitediffs',
'description': 'differing pixels in white',
},
},
}
image_pair_set = imagepairset.ImagePairSet(
descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION),
diff_base_url=DIFF_BASE_URL)
for image_pair in image_pairs:
image_pair_set.add_image_pair(image_pair)
# The 'builder' column header uses the default settings,
# but the 'test' column header has manual adjustments.
image_pair_set.set_column_header_factory(
'test',
column.ColumnHeaderFactory(
header_text='which GM test',
header_url='http://learn/about/gm/tests',
is_filterable=True,
is_sortable=False))
self.assertEqual(image_pair_set.as_dict(), expected_imageset_dict)
def _load_result_pairs(self, setA_root, setB_root, setA_section,
setB_section):
"""Loads all JSON image summaries from 2 directory trees and compares them.
TODO(stephana): This method is only called from within __init__(); it might
make more sense to just roll the content of this method into __init__().
Args:
setA_root: root directory containing JSON summaries of rendering results
setB_root: root directory containing JSON summaries of rendering results
setA_section: which section (gm_json.JSONKEY_ACTUALRESULTS or
gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setA
setB_section: which section (gm_json.JSONKEY_ACTUALRESULTS or
gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setB
Returns the summary of all image diff results (or None, depending on
self._prefetch_only).
"""
logging.info('Reading JSON image summaries from dirs %s and %s...' %
(setA_root, setB_root))
setA_dicts = self.read_dicts_from_root(setA_root)
setB_dicts = self.read_dicts_from_root(setB_root)
logging.info('Comparing summary dicts...')
all_image_pairs = imagepairset.ImagePairSet(
descriptions=(self._setA_label, self._setB_label),
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=(self._setA_label, self._setB_label),
diff_base_url=self._diff_base_url)
# Override settings for columns that should be filtered using freeform text.
for column_id in FREEFORM_COLUMN_IDS:
factory = column.ColumnHeaderFactory(header_text=column_id,
use_freeform_filter=True)
all_image_pairs.set_column_header_factory(
column_id=column_id, column_header_factory=factory)
failing_image_pairs.set_column_header_factory(
column_id=column_id, column_header_factory=factory)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=COLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=COLUMN__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
logging.info('Starting to add imagepairs to queue.')
self._image_diff_db.log_queue_size_if_changed(limit_verbosity=False)
union_dict_paths = sorted(set(setA_dicts.keys() + setB_dicts.keys()))
num_union_dict_paths = len(union_dict_paths)
dict_num = 0
for dict_path in union_dict_paths:
dict_num += 1
logging.info(
'Asynchronously requesting pixel diffs for dict #%d of %d, "%s"...'
% (dict_num, num_union_dict_paths, dict_path))
dictA = self.get_default(setA_dicts, None, dict_path)
self._validate_dict_version(dictA)
dictA_results = self.get_default(dictA, {}, setA_section)
dictB = self.get_default(setB_dicts, None, dict_path)
self._validate_dict_version(dictB)
dictB_results = self.get_default(dictB, {}, setB_section)
image_A_base_url = self.get_default(
setA_dicts, self._image_base_gs_url, dict_path,
gm_json.JSONKEY_IMAGE_BASE_GS_URL)
image_B_base_url = self.get_default(
setB_dicts, self._image_base_gs_url, dict_path,
gm_json.JSONKEY_IMAGE_BASE_GS_URL)
# get the builders and render modes for each set
builder_A = self.get_default(dictA, None,
gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_BUILDER)
render_mode_A = self.get_default(
dictA, None, gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_RENDER_MODE)
builder_B = self.get_default(dictB, None,
gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_BUILDER)
render_mode_B = self.get_default(
dictB, None, gm_json.JSONKEY_DESCRIPTIONS,
gm_json.JSONKEY_DESCRIPTIONS_RENDER_MODE)
skp_names = sorted(set(dictA_results.keys() +
dictB_results.keys()))
# Just for manual testing... truncate to an arbitrary subset.
if self.truncate_results:
skp_names = skp_names[1:3]
for skp_name in skp_names:
imagepairs_for_this_skp = []
whole_image_A = self.get_default(
dictA_results, None, skp_name,
gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
whole_image_B = self.get_default(
dictB_results, None, skp_name,
gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
imagepairs_for_this_skp.append(
self._create_image_pair(image_dict_A=whole_image_A,
image_dict_B=whole_image_B,
image_A_base_url=image_A_base_url,
image_B_base_url=image_B_base_url,
builder_A=builder_A,
render_mode_A=render_mode_A,
builder_B=builder_B,
render_mode_B=render_mode_B,
source_json_file=dict_path,
source_skp_name=skp_name,
tilenum=None))
tiled_images_A = self.get_default(
dictA_results, [], skp_name,
gm_json.JSONKEY_SOURCE_TILEDIMAGES)
tiled_images_B = self.get_default(
dictB_results, [], skp_name,
gm_json.JSONKEY_SOURCE_TILEDIMAGES)
if tiled_images_A or tiled_images_B:
num_tiles_A = len(tiled_images_A)
num_tiles_B = len(tiled_images_B)
num_tiles = max(num_tiles_A, num_tiles_B)
for tile_num in range(num_tiles):
imagepairs_for_this_skp.append(
self._create_image_pair(
image_dict_A=(tiled_images_A[tile_num]
if tile_num < num_tiles_A else
None),
image_dict_B=(tiled_images_B[tile_num]
if tile_num < num_tiles_B else
None),
image_A_base_url=image_A_base_url,
image_B_base_url=image_B_base_url,
builder_A=builder_A,
render_mode_A=render_mode_A,
builder_B=builder_B,
render_mode_B=render_mode_B,
source_json_file=dict_path,
source_skp_name=skp_name,
tilenum=tile_num))
for one_imagepair in imagepairs_for_this_skp:
if one_imagepair:
all_image_pairs.add_image_pair(one_imagepair)
result_type = one_imagepair.extra_columns_dict\
[COLUMN__RESULT_TYPE]
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(one_imagepair)
logging.info('Finished adding imagepairs to queue.')
self._image_diff_db.log_queue_size_if_changed(limit_verbosity=False)
if self._prefetch_only:
return None
else:
return {
results.KEY__HEADER__RESULTS_ALL:
all_image_pairs.as_dict(
column_ids_in_order=ORDERED_COLUMN_IDS),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(
column_ids_in_order=ORDERED_COLUMN_IDS),
}
def _load_actual_and_expected(self):
"""Loads the results of all tests, across all builders (based on the
files within self._actuals_root and self._expected_root),
and stores them in self._results.
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
actual_builder_dicts = self._read_builder_dicts_from_root(
self._actuals_root)
logging.info('Reading expected-results JSON files from %s...' %
self._expected_root)
expected_builder_dicts = self._read_builder_dicts_from_root(
self._expected_root)
all_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff_base_url=self._diff_base_url)
# Override settings for columns that should be filtered using freeform text.
for column_id in FREEFORM_COLUMN_IDS:
factory = column.ColumnHeaderFactory(header_text=column_id,
use_freeform_filter=True)
all_image_pairs.set_column_header_factory(
column_id=column_id, column_header_factory=factory)
failing_image_pairs.set_column_header_factory(
column_id=column_id, column_header_factory=factory)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__FAILUREIGNORED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE,
values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__FAILUREIGNORED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
# Only consider builders we have both expected and actual results for.
# Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
# (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
# builder')
actual_builder_set = set(actual_builder_dicts.keys())
expected_builder_set = set(expected_builder_dicts.keys())
builders = sorted(
actual_builder_set.intersection(expected_builder_set))
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info(
'Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
actual_results_for_this_builder = (
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[
result_type]
if not results_of_this_type:
continue
for image_name in sorted(results_of_this_type.keys()):
(test, config
) = results.IMAGE_FILENAME_RE.match(image_name).groups()
actual_image_relative_url = (
ExpectationComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type[
image_name],
test_name=test))
# Default empty expectations; overwrite these if we find any real ones
expectations_per_test = None
expected_image_relative_url = None
expectations_dict = None
try:
expectations_per_test = (
expected_builder_dicts[builder][
gm_json.JSONKEY_EXPECTEDRESULTS][image_name])
# TODO(epoger): assumes a single allowed digest per test, which is
# fine; see https://code.google.com/p/skia/issues/detail?id=1787
expected_image_hashtype_and_digest = (
expectations_per_test[
gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
[0])
expected_image_relative_url = (
ExpectationComparisons._create_relative_url(
hashtype_and_digest=
expected_image_hashtype_and_digest,
test_name=test))
expectations_dict = {}
for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
expectations_dict[
field] = expectations_per_test.get(field)
except (KeyError, TypeError):
# There are several cases in which we would expect to find
# no expectations for a given test:
#
# 1. result_type == NOCOMPARISON
# There are no expectations for this test yet!
#
# 2. alternate rendering mode failures (e.g. serialized)
# In cases like
# https://code.google.com/p/skia/issues/detail?id=1684
# ('tileimagefilter GM test failing in serialized render mode'),
# the gm-actuals will list a failure for the alternate
# rendering mode even though we don't have explicit expectations
# for the test (the implicit expectation is that it must
# render the same in all rendering modes).
#
# Don't log type 1, because it is common.
# Log other types, because they are rare and we should know about
# them, but don't throw an exception, because we need to keep our
# tools working in the meanwhile!
if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
logging.warning(
'No expectations found for test: %s' % {
results.KEY__EXTRACOLUMNS__BUILDER:
builder,
results.KEY__EXTRACOLUMNS__RESULT_TYPE:
result_type,
'image_name': image_name,
})
# If this test was recently rebaselined, it will remain in
# the 'failed' set of actuals until all the bots have
# cycled (although the expectations have indeed been set
# from the most recent actuals). Treat these as successes
# instead of failures.
#
# TODO(epoger): Do we need to do something similar in
# other cases, such as when we have recently marked a test
# as ignoreFailure but it still shows up in the 'failed'
# category? Maybe we should not rely on the result_type
# categories recorded within the gm_actuals AT ALL, and
# instead evaluate the result_type ourselves based on what
# we see in expectations vs actual checksum?
if expected_image_relative_url == actual_image_relative_url:
updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif ((result_type == results.KEY__RESULT_TYPE__FAILED)
and (test in self._ignore_failures_on_these_tests)):
updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
else:
updated_result_type = result_type
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__RESULT_TYPE:
updated_result_type,
results.KEY__EXTRACOLUMNS__BUILDER: builder,
results.KEY__EXTRACOLUMNS__TEST: test,
results.KEY__EXTRACOLUMNS__CONFIG: config,
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageA_relative_url=expected_image_relative_url,
imageB_relative_url=actual_image_relative_url,
expectations=expectations_dict,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except Exception:
logging.exception(
'got exception while creating new ImagePair')
# pylint: disable=W0201
self._results = {
results.KEY__HEADER__RESULTS_ALL:
all_image_pairs.as_dict(column_ids_in_order=ORDERED_COLUMN_IDS),
results.KEY__HEADER__RESULTS_FAILURES:
failing_image_pairs.as_dict(
column_ids_in_order=ORDERED_COLUMN_IDS),
}