def test_success(self): """Assembles some ImagePairs into an ImagePairSet, and validates results. """ image_pairs = [ MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_1_AS_DICT), MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_2_AS_DICT), MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_3_AS_DICT), ] expected_imageset_dict = { 'extraColumnHeaders': { 'builder': { 'headerText': 'builder', 'isFilterable': True, 'isSortable': True, 'valuesAndCounts': { 'MyBuilder': 3 }, }, 'test': { 'headerText': 'which GM test', 'headerUrl': 'http://learn/about/gm/tests', 'isFilterable': True, 'isSortable': False, }, }, 'imagePairs': [ IMAGEPAIR_1_AS_DICT, IMAGEPAIR_2_AS_DICT, IMAGEPAIR_3_AS_DICT, ], 'imageSets': [ { 'baseUrl': BASE_URL_1, 'description': SET_A_DESCRIPTION, }, { 'baseUrl': BASE_URL_1, 'description': SET_B_DESCRIPTION, }, ], } image_pair_set = imagepairset.ImagePairSet( descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION)) for image_pair in image_pairs: image_pair_set.add_image_pair(image_pair) # The 'builder' column header uses the default settings, # but the 'test' column header has manual adjustments. image_pair_set.set_column_header_factory( 'test', column.ColumnHeaderFactory( header_text='which GM test', header_url='http://learn/about/gm/tests', is_filterable=True, is_sortable=False, include_values_and_counts=False)) self.assertEqual(image_pair_set.as_dict(), expected_imageset_dict)
def test_mismatched_base_url(self): """Confirms that mismatched base_urls will cause an exception.""" image_pair_set = imagepairset.ImagePairSet( diff_base_url=DIFF_BASE_URL) image_pair_set.add_image_pair( MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_1_AS_DICT)) image_pair_set.add_image_pair( MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_2_AS_DICT)) with self.assertRaises(Exception): image_pair_set.add_image_pair( MockImagePair(base_url=BASE_URL_2, dict_to_return=IMAGEPAIR_3_AS_DICT))
def test_missing_column_ids(self): """Confirms that passing truncated column_ids_in_order to as_dict() will cause an exception.""" image_pair_set = imagepairset.ImagePairSet(diff_base_url=DIFF_BASE_URL) image_pair_set.add_image_pair( MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_1_AS_DICT)) image_pair_set.add_image_pair( MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_2_AS_DICT)) # Call as_dict() with default or reasonable column_ids_in_order. image_pair_set.as_dict() image_pair_set.as_dict(column_ids_in_order=['test', 'builder']) image_pair_set.as_dict( column_ids_in_order=['test', 'builder', 'extra']) # Call as_dict() with not enough column_ids. with self.assertRaises(Exception): image_pair_set.as_dict(column_ids_in_order=['builder'])
def _load_config_pairs(self, configs): """Loads the results of all tests, across all builders (based on the files within self._actuals_root), compares them across two configs, and stores the summary in self._results. Args: configs: tuple of strings; pair of configs to compare """ logging.info('Reading actual-results JSON files from %s...' % self._actuals_root) actual_builder_dicts = self._read_builder_dicts_from_root( self._actuals_root) configA, configB = configs logging.info('Comparing configs %s and %s...' % (configA, configB)) all_image_pairs = imagepairset.ImagePairSet( descriptions=configs, diff_base_url=self._diff_base_url) failing_image_pairs = imagepairset.ImagePairSet( descriptions=configs, diff_base_url=self._diff_base_url) all_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, results.KEY__RESULT_TYPE__SUCCEEDED, ]) failing_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, ]) builders = sorted(actual_builder_dicts.keys()) num_builders = len(builders) builder_num = 0 for builder in builders: builder_num += 1 logging.info( 'Generating pixel diffs for builder #%d of %d, "%s"...' % (builder_num, num_builders, builder)) actual_results_for_this_builder = ( actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) for result_type in sorted(actual_results_for_this_builder.keys()): results_of_this_type = actual_results_for_this_builder[ result_type] if not results_of_this_type: continue tests_found = set() for image_name in sorted(results_of_this_type.keys()): (test, _) = results.IMAGE_FILENAME_RE.match(image_name).groups() tests_found.add(test) for test in tests_found: # Get image_relative_url (or None) for each of configA, configB image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA) configA_image_relative_url = ConfigComparisons._create_relative_url( hashtype_and_digest=results_of_this_type.get( image_name_A), test_name=test) image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB) configB_image_relative_url = ConfigComparisons._create_relative_url( hashtype_and_digest=results_of_this_type.get( image_name_B), test_name=test) # If we have images for at least one of these two configs, # add them to our list. if configA_image_relative_url or configB_image_relative_url: if configA_image_relative_url == configB_image_relative_url: result_type = results.KEY__RESULT_TYPE__SUCCEEDED elif not configA_image_relative_url: result_type = results.KEY__RESULT_TYPE__NOCOMPARISON elif not configB_image_relative_url: result_type = results.KEY__RESULT_TYPE__NOCOMPARISON else: result_type = results.KEY__RESULT_TYPE__FAILED extra_columns_dict = { results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, results.KEY__EXTRACOLUMNS__BUILDER: builder, results.KEY__EXTRACOLUMNS__TEST: test, # TODO(epoger): Right now, the client UI crashes if it receives # results that do not include a 'config' column. # Until we fix that, keep the client happy. results.KEY__EXTRACOLUMNS__CONFIG: 'TODO', } try: image_pair = imagepair.ImagePair( image_diff_db=self._image_diff_db, base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, imageA_relative_url=configA_image_relative_url, imageB_relative_url=configB_image_relative_url, extra_columns=extra_columns_dict) all_image_pairs.add_image_pair(image_pair) if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: failing_image_pairs.add_image_pair(image_pair) except (KeyError, TypeError): logging.exception( 'got exception while creating ImagePair for test ' '"%s", builder "%s"' % (test, builder)) # pylint: disable=W0201 self._results = { results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), }
def test_success(self): """Assembles some ImagePairs into an ImagePairSet, and validates results. """ image_pairs = [ MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_1_AS_DICT), MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_2_AS_DICT), MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_3_AS_DICT), ] expected_imageset_dict = { 'extraColumnHeaders': { 'builder': { 'headerText': 'builder', 'isFilterable': True, 'isSortable': True, 'useFreeformFilter': False, 'valuesAndCounts': [('MyBuilder', 3)], }, 'test': { 'headerText': 'which GM test', 'headerUrl': 'http://learn/about/gm/tests', 'isFilterable': True, 'isSortable': False, 'useFreeformFilter': False, 'valuesAndCounts': [('test1', 1), ('test2', 1), ('test3', 1)], }, }, 'extraColumnOrder': ['builder', 'test'], 'imagePairs': [ IMAGEPAIR_1_AS_DICT, IMAGEPAIR_2_AS_DICT, IMAGEPAIR_3_AS_DICT, ], 'imageSets': { 'imageA': { 'baseUrl': BASE_URL_1, 'description': SET_A_DESCRIPTION, }, 'imageB': { 'baseUrl': BASE_URL_1, 'description': SET_B_DESCRIPTION, }, 'diffs': { 'baseUrl': DIFF_BASE_URL + '/diffs', 'description': 'color difference per channel', }, 'whiteDiffs': { 'baseUrl': DIFF_BASE_URL + '/whitediffs', 'description': 'differing pixels in white', }, }, } image_pair_set = imagepairset.ImagePairSet( descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION), diff_base_url=DIFF_BASE_URL) for image_pair in image_pairs: image_pair_set.add_image_pair(image_pair) # The 'builder' column header uses the default settings, # but the 'test' column header has manual adjustments. image_pair_set.set_column_header_factory( 'test', column.ColumnHeaderFactory( header_text='which GM test', header_url='http://learn/about/gm/tests', is_filterable=True, is_sortable=False)) self.assertEqual(image_pair_set.as_dict(), expected_imageset_dict)
def _load_actual_and_expected(self): """Loads the results of all tests, across all builders (based on the files within self._actuals_root and self._expected_root), and stores them in self._results. """ logging.info('Reading actual-results JSON files from %s...' % self._actuals_root) actual_builder_dicts = self._read_builder_dicts_from_root( self._actuals_root) logging.info('Reading expected-results JSON files from %s...' % self._expected_root) expected_builder_dicts = self._read_builder_dicts_from_root( self._expected_root) all_image_pairs = imagepairset.ImagePairSet( descriptions=IMAGEPAIR_SET_DESCRIPTIONS, diff_base_url=self._diff_base_url) failing_image_pairs = imagepairset.ImagePairSet( descriptions=IMAGEPAIR_SET_DESCRIPTIONS, diff_base_url=self._diff_base_url) all_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__FAILUREIGNORED, results.KEY__RESULT_TYPE__NOCOMPARISON, results.KEY__RESULT_TYPE__SUCCEEDED, ]) failing_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__FAILUREIGNORED, results.KEY__RESULT_TYPE__NOCOMPARISON, ]) # Only consider builders we have both expected and actual results for. # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug # builder') actual_builder_set = set(actual_builder_dicts.keys()) expected_builder_set = set(expected_builder_dicts.keys()) builders = sorted(actual_builder_set.intersection(expected_builder_set)) num_builders = len(builders) builder_num = 0 for builder in builders: builder_num += 1 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % (builder_num, num_builders, builder)) actual_results_for_this_builder = ( actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) for result_type in sorted(actual_results_for_this_builder.keys()): results_of_this_type = actual_results_for_this_builder[result_type] if not results_of_this_type: continue for image_name in sorted(results_of_this_type.keys()): (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() actual_image_relative_url = ( ExpectationComparisons._create_relative_url( hashtype_and_digest=results_of_this_type[image_name], test_name=test)) # Default empty expectations; overwrite these if we find any real ones expectations_per_test = None expected_image_relative_url = None expectations_dict = None try: expectations_per_test = ( expected_builder_dicts [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) # TODO(epoger): assumes a single allowed digest per test, which is # fine; see https://code.google.com/p/skia/issues/detail?id=1787 expected_image_hashtype_and_digest = ( expectations_per_test [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) expected_image_relative_url = ( ExpectationComparisons._create_relative_url( hashtype_and_digest=expected_image_hashtype_and_digest, test_name=test)) expectations_dict = {} for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: expectations_dict[field] = expectations_per_test.get(field) except (KeyError, TypeError): # There are several cases in which we would expect to find # no expectations for a given test: # # 1. result_type == NOCOMPARISON # There are no expectations for this test yet! # # 2. alternate rendering mode failures (e.g. serialized) # In cases like # https://code.google.com/p/skia/issues/detail?id=1684 # ('tileimagefilter GM test failing in serialized render mode'), # the gm-actuals will list a failure for the alternate # rendering mode even though we don't have explicit expectations # for the test (the implicit expectation is that it must # render the same in all rendering modes). # # Don't log type 1, because it is common. # Log other types, because they are rare and we should know about # them, but don't throw an exception, because we need to keep our # tools working in the meanwhile! if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: logging.warning('No expectations found for test: %s' % { results.KEY__EXTRACOLUMNS__BUILDER: builder, results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, 'image_name': image_name, }) # If this test was recently rebaselined, it will remain in # the 'failed' set of actuals until all the bots have # cycled (although the expectations have indeed been set # from the most recent actuals). Treat these as successes # instead of failures. # # TODO(epoger): Do we need to do something similar in # other cases, such as when we have recently marked a test # as ignoreFailure but it still shows up in the 'failed' # category? Maybe we should not rely on the result_type # categories recorded within the gm_actuals AT ALL, and # instead evaluate the result_type ourselves based on what # we see in expectations vs actual checksum? if expected_image_relative_url == actual_image_relative_url: updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and (test in self._ignore_failures_on_these_tests)): updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED else: updated_result_type = result_type extra_columns_dict = { results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type, results.KEY__EXTRACOLUMNS__BUILDER: builder, results.KEY__EXTRACOLUMNS__TEST: test, results.KEY__EXTRACOLUMNS__CONFIG: config, } try: image_pair = imagepair.ImagePair( image_diff_db=self._image_diff_db, base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, imageA_relative_url=expected_image_relative_url, imageB_relative_url=actual_image_relative_url, expectations=expectations_dict, extra_columns=extra_columns_dict) all_image_pairs.add_image_pair(image_pair) if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: failing_image_pairs.add_image_pair(image_pair) except Exception: logging.exception('got exception while creating new ImagePair') self._results = { results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), }
def _load_result_pairs(self, actuals_root, subdirs): """Loads all JSON files found within two subdirs in actuals_root, compares across those two subdirs, and stores the summary in self._results. Args: actuals_root: root directory containing all render_pictures-generated JSON files subdirs: (string, string) tuple; pair of subdirectories within actuals_root to compare """ logging.info( 'Reading actual-results JSON files from %s subdirs within %s...' % (subdirs, actuals_root)) subdirA, subdirB = subdirs subdirA_builder_dicts = self._read_dicts_from_root( os.path.join(actuals_root, subdirA)) subdirB_builder_dicts = self._read_dicts_from_root( os.path.join(actuals_root, subdirB)) logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB)) all_image_pairs = imagepairset.ImagePairSet( descriptions=subdirs, diff_base_url=self._diff_base_url) failing_image_pairs = imagepairset.ImagePairSet( descriptions=subdirs, diff_base_url=self._diff_base_url) all_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, results.KEY__RESULT_TYPE__SUCCEEDED, ]) failing_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, ]) builders = sorted( set(subdirA_builder_dicts.keys() + subdirB_builder_dicts.keys())) num_builders = len(builders) builder_num = 0 for builder in builders: builder_num += 1 logging.info( 'Generating pixel diffs for builder #%d of %d, "%s"...' % (builder_num, num_builders, builder)) # TODO(epoger): This will fail if we have results for this builder in # subdirA but not subdirB (or vice versa). subdirA_results = results.BaseComparisons.combine_subdicts( subdirA_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) subdirB_results = results.BaseComparisons.combine_subdicts( subdirB_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) image_names = sorted( set(subdirA_results.keys() + subdirB_results.keys())) for image_name in image_names: # The image name may contain funny characters or be ridiculously long # (see https://code.google.com/p/skia/issues/detail?id=2344#c10 ), # so make sure we sanitize it before using it in a URL path. # # TODO(epoger): Rather than sanitizing/truncating the image name here, # do it in render_pictures instead. # Reason: we will need to be consistent in applying this rule, so that # the process which uploads the files to GS using these paths will # match the paths created by downstream processes. # So, we should make render_pictures write out images to paths that are # "ready to upload" to Google Storage, like gm does. sanitized_test_name = DISALLOWED_FILEPATH_CHAR_REGEX.sub( '_', image_name)[:30] subdirA_image_relative_url = ( results.BaseComparisons._create_relative_url( hashtype_and_digest=subdirA_results.get(image_name), test_name=sanitized_test_name)) subdirB_image_relative_url = ( results.BaseComparisons._create_relative_url( hashtype_and_digest=subdirB_results.get(image_name), test_name=sanitized_test_name)) # If we have images for at least one of these two subdirs, # add them to our list. if subdirA_image_relative_url or subdirB_image_relative_url: if subdirA_image_relative_url == subdirB_image_relative_url: result_type = results.KEY__RESULT_TYPE__SUCCEEDED elif not subdirA_image_relative_url: result_type = results.KEY__RESULT_TYPE__NOCOMPARISON elif not subdirB_image_relative_url: result_type = results.KEY__RESULT_TYPE__NOCOMPARISON else: result_type = results.KEY__RESULT_TYPE__FAILED extra_columns_dict = { results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type, results.KEY__EXTRACOLUMN__BUILDER: builder, results.KEY__EXTRACOLUMN__TEST: image_name, # TODO(epoger): Right now, the client UI crashes if it receives # results that do not include a 'config' column. # Until we fix that, keep the client happy. results.KEY__EXTRACOLUMN__CONFIG: 'TODO', } try: image_pair = imagepair.ImagePair( image_diff_db=self._image_diff_db, base_url=self._image_base_url, imageA_relative_url=subdirA_image_relative_url, imageB_relative_url=subdirB_image_relative_url, extra_columns=extra_columns_dict) all_image_pairs.add_image_pair(image_pair) if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: failing_image_pairs.add_image_pair(image_pair) except (KeyError, TypeError): logging.exception( 'got exception while creating ImagePair for image_name ' '"%s", builder "%s"' % (image_name, builder)) self._results = { results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), }
def _load_result_pairs(self, setA_root, setB_root, setA_section, setB_section): """Loads all JSON image summaries from 2 directory trees and compares them. TODO(stephana): This method is only called from within __init__(); it might make more sense to just roll the content of this method into __init__(). Args: setA_root: root directory containing JSON summaries of rendering results setB_root: root directory containing JSON summaries of rendering results setA_section: which section (gm_json.JSONKEY_ACTUALRESULTS or gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setA setB_section: which section (gm_json.JSONKEY_ACTUALRESULTS or gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setB Returns the summary of all image diff results (or None, depending on self._prefetch_only). """ logging.info('Reading JSON image summaries from dirs %s and %s...' % (setA_root, setB_root)) setA_dicts = self.read_dicts_from_root(setA_root) setB_dicts = self.read_dicts_from_root(setB_root) logging.info('Comparing summary dicts...') all_image_pairs = imagepairset.ImagePairSet( descriptions=(self._setA_label, self._setB_label), diff_base_url=self._diff_base_url) failing_image_pairs = imagepairset.ImagePairSet( descriptions=(self._setA_label, self._setB_label), diff_base_url=self._diff_base_url) # Override settings for columns that should be filtered using freeform text. for column_id in FREEFORM_COLUMN_IDS: factory = column.ColumnHeaderFactory(header_text=column_id, use_freeform_filter=True) all_image_pairs.set_column_header_factory( column_id=column_id, column_header_factory=factory) failing_image_pairs.set_column_header_factory( column_id=column_id, column_header_factory=factory) all_image_pairs.ensure_extra_column_values_in_summary( column_id=COLUMN__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, results.KEY__RESULT_TYPE__SUCCEEDED, ]) failing_image_pairs.ensure_extra_column_values_in_summary( column_id=COLUMN__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, ]) logging.info('Starting to add imagepairs to queue.') self._image_diff_db.log_queue_size_if_changed(limit_verbosity=False) union_dict_paths = sorted(set(setA_dicts.keys() + setB_dicts.keys())) num_union_dict_paths = len(union_dict_paths) dict_num = 0 for dict_path in union_dict_paths: dict_num += 1 logging.info( 'Asynchronously requesting pixel diffs for dict #%d of %d, "%s"...' % (dict_num, num_union_dict_paths, dict_path)) dictA = self.get_default(setA_dicts, None, dict_path) self._validate_dict_version(dictA) dictA_results = self.get_default(dictA, {}, setA_section) dictB = self.get_default(setB_dicts, None, dict_path) self._validate_dict_version(dictB) dictB_results = self.get_default(dictB, {}, setB_section) image_A_base_url = self.get_default( setA_dicts, self._image_base_gs_url, dict_path, gm_json.JSONKEY_IMAGE_BASE_GS_URL) image_B_base_url = self.get_default( setB_dicts, self._image_base_gs_url, dict_path, gm_json.JSONKEY_IMAGE_BASE_GS_URL) # get the builders and render modes for each set builder_A = self.get_default(dictA, None, gm_json.JSONKEY_DESCRIPTIONS, gm_json.JSONKEY_DESCRIPTIONS_BUILDER) render_mode_A = self.get_default( dictA, None, gm_json.JSONKEY_DESCRIPTIONS, gm_json.JSONKEY_DESCRIPTIONS_RENDER_MODE) builder_B = self.get_default(dictB, None, gm_json.JSONKEY_DESCRIPTIONS, gm_json.JSONKEY_DESCRIPTIONS_BUILDER) render_mode_B = self.get_default( dictB, None, gm_json.JSONKEY_DESCRIPTIONS, gm_json.JSONKEY_DESCRIPTIONS_RENDER_MODE) skp_names = sorted(set(dictA_results.keys() + dictB_results.keys())) # Just for manual testing... truncate to an arbitrary subset. if self.truncate_results: skp_names = skp_names[1:3] for skp_name in skp_names: imagepairs_for_this_skp = [] whole_image_A = self.get_default( dictA_results, None, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) whole_image_B = self.get_default( dictB_results, None, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) imagepairs_for_this_skp.append( self._create_image_pair(image_dict_A=whole_image_A, image_dict_B=whole_image_B, image_A_base_url=image_A_base_url, image_B_base_url=image_B_base_url, builder_A=builder_A, render_mode_A=render_mode_A, builder_B=builder_B, render_mode_B=render_mode_B, source_json_file=dict_path, source_skp_name=skp_name, tilenum=None)) tiled_images_A = self.get_default( dictA_results, [], skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) tiled_images_B = self.get_default( dictB_results, [], skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) if tiled_images_A or tiled_images_B: num_tiles_A = len(tiled_images_A) num_tiles_B = len(tiled_images_B) num_tiles = max(num_tiles_A, num_tiles_B) for tile_num in range(num_tiles): imagepairs_for_this_skp.append( self._create_image_pair( image_dict_A=(tiled_images_A[tile_num] if tile_num < num_tiles_A else None), image_dict_B=(tiled_images_B[tile_num] if tile_num < num_tiles_B else None), image_A_base_url=image_A_base_url, image_B_base_url=image_B_base_url, builder_A=builder_A, render_mode_A=render_mode_A, builder_B=builder_B, render_mode_B=render_mode_B, source_json_file=dict_path, source_skp_name=skp_name, tilenum=tile_num)) for one_imagepair in imagepairs_for_this_skp: if one_imagepair: all_image_pairs.add_image_pair(one_imagepair) result_type = one_imagepair.extra_columns_dict\ [COLUMN__RESULT_TYPE] if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: failing_image_pairs.add_image_pair(one_imagepair) logging.info('Finished adding imagepairs to queue.') self._image_diff_db.log_queue_size_if_changed(limit_verbosity=False) if self._prefetch_only: return None else: return { results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict( column_ids_in_order=ORDERED_COLUMN_IDS), results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict( column_ids_in_order=ORDERED_COLUMN_IDS), }
def _load_result_pairs(self, actuals_root, subdirs): """Loads all JSON files found within two subdirs in actuals_root, compares across those two subdirs, and stores the summary in self._results. Args: actuals_root: root directory containing all render_pictures-generated JSON files subdirs: (string, string) tuple; pair of subdirectories within actuals_root to compare """ logging.info( 'Reading actual-results JSON files from %s subdirs within %s...' % ( subdirs, actuals_root)) subdirA, subdirB = subdirs subdirA_dicts = self._read_dicts_from_root( os.path.join(actuals_root, subdirA)) subdirB_dicts = self._read_dicts_from_root( os.path.join(actuals_root, subdirB)) logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB)) all_image_pairs = imagepairset.ImagePairSet( descriptions=subdirs, diff_base_url=self._diff_base_url) failing_image_pairs = imagepairset.ImagePairSet( descriptions=subdirs, diff_base_url=self._diff_base_url) all_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, results.KEY__RESULT_TYPE__SUCCEEDED, ]) failing_image_pairs.ensure_extra_column_values_in_summary( column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, ]) common_dict_paths = sorted(set(subdirA_dicts.keys() + subdirB_dicts.keys())) num_common_dict_paths = len(common_dict_paths) dict_num = 0 for dict_path in common_dict_paths: dict_num += 1 logging.info('Generating pixel diffs for dict #%d of %d, "%s"...' % (dict_num, num_common_dict_paths, dict_path)) dictA = subdirA_dicts[dict_path] dictB = subdirB_dicts[dict_path] self._validate_dict_version(dictA) self._validate_dict_version(dictB) dictA_results = dictA[gm_json.JSONKEY_ACTUALRESULTS] dictB_results = dictB[gm_json.JSONKEY_ACTUALRESULTS] skp_names = sorted(set(dictA_results.keys() + dictB_results.keys())) for skp_name in skp_names: imagepairs_for_this_skp = [] whole_image_A = RenderedPicturesComparisons.get_multilevel( dictA_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) whole_image_B = RenderedPicturesComparisons.get_multilevel( dictB_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) imagepairs_for_this_skp.append(self._create_image_pair( test=skp_name, config=gm_json.JSONKEY_SOURCE_WHOLEIMAGE, image_dict_A=whole_image_A, image_dict_B=whole_image_B)) tiled_images_A = RenderedPicturesComparisons.get_multilevel( dictA_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) tiled_images_B = RenderedPicturesComparisons.get_multilevel( dictB_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) # TODO(epoger): Report an error if we find tiles for A but not B? if tiled_images_A and tiled_images_B: # TODO(epoger): Report an error if we find a different number of tiles # for A and B? num_tiles = len(tiled_images_A) for tile_num in range(num_tiles): imagepairs_for_this_skp.append(self._create_image_pair( test=skp_name, config='%s-%d' % (gm_json.JSONKEY_SOURCE_TILEDIMAGES, tile_num), image_dict_A=tiled_images_A[tile_num], image_dict_B=tiled_images_B[tile_num])) for imagepair in imagepairs_for_this_skp: if imagepair: all_image_pairs.add_image_pair(imagepair) result_type = imagepair.extra_columns_dict\ [results.KEY__EXTRACOLUMNS__RESULT_TYPE] if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: failing_image_pairs.add_image_pair(imagepair) self._results = { results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), }