def test_gm(self):
"""Process results of a GM run with the ExpectationComparisons object."""
results_obj = compare_to_expectations.ExpectationComparisons(
actuals_root=os.path.join(self._input_dir, 'gm-actuals'),
expected_root=os.path.join(self._input_dir, 'gm-expectations'),
generated_images_root=self._temp_dir,
diff_base_url='/static/generated-images')
results_obj.get_timestamp = mock_get_timestamp
gm_json.WriteToFile(
results_obj.get_packaged_results_of_type(
results.KEY__HEADER__RESULTS_ALL),
os.path.join(self._output_dir_actual, 'gm.json'))
def update_results(self, invalidate=False):
""" Create or update self._results, based on the latest expectations and
actuals.
We hold self.results_rlock while we do this, to guarantee that no other
thread attempts to update either self._results or the underlying files at
the same time.
Args:
invalidate: if True, invalidate self._results immediately upon entry;
otherwise, we will let readers see those results until we
replace them
"""
with self.results_rlock:
if invalidate:
self._results = None
if self._actuals_repo_url:
logging.info(
'Updating actual GM results in %s to revision %s from repo %s ...'
% (self._actuals_dir, self._actuals_repo_revision,
self._actuals_repo_url))
self._actuals_repo.Update(path='.',
revision=self._actuals_repo_revision)
# We only update the expectations dir if the server was run with a
# nonzero --reload argument; otherwise, we expect the user to maintain
# her own expectations as she sees fit.
#
# Because the Skia repo is moving from SVN to git, and git does not
# support updating a single directory tree, we have to update the entire
# repo checkout.
#
# Because Skia uses depot_tools, we have to update using "gclient sync"
# instead of raw git (or SVN) update. Happily, this will work whether
# the checkout was created using git or SVN.
if self._reload_seconds:
logging.info(
'Updating expected GM results in %s by syncing Skia repo ...'
% compare_to_expectations.DEFAULT_EXPECTATIONS_DIR)
_run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
self._results = compare_to_expectations.ExpectationComparisons(
actuals_root=self._actuals_dir,
generated_images_root=os.path.join(PARENT_DIRECTORY,
STATIC_CONTENTS_SUBDIR,
GENERATED_IMAGES_SUBDIR),
diff_base_url=posixpath.join(os.pardir, STATIC_CONTENTS_SUBDIR,
GENERATED_IMAGES_SUBDIR),
builder_regex_list=self._builder_regex_list)
json_dir = os.path.join(PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
GENERATED_JSON_SUBDIR)
if not os.path.isdir(json_dir):
os.makedirs(json_dir)
for config_pair in self._config_pairs:
config_comparisons = compare_configs.ConfigComparisons(
configs=config_pair,
actuals_root=self._actuals_dir,
generated_images_root=os.path.join(
PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
GENERATED_IMAGES_SUBDIR),
diff_base_url=posixpath.join(os.pardir,
GENERATED_IMAGES_SUBDIR),
builder_regex_list=self._builder_regex_list)
for summary_type in SUMMARY_TYPES:
gm_json.WriteToFile(
config_comparisons.get_packaged_results_of_type(
results_type=summary_type),
os.path.join(
json_dir, '%s-vs-%s_%s.json' %
(config_pair[0], config_pair[1], summary_type)))
def update_results(self, invalidate=False):
""" Create or update self._results, based on the latest expectations and
actuals.
We hold self.results_rlock while we do this, to guarantee that no other
thread attempts to update either self._results or the underlying files at
the same time.
Args:
invalidate: if True, invalidate self._results immediately upon entry;
otherwise, we will let readers see those results until we
replace them
"""
with self.results_rlock:
if invalidate:
self._results = None
if self._gm_summaries_bucket:
logging.info(
'Updating GM result summaries in %s from gm_summaries_bucket %s ...'
% (self._actuals_dir, self._gm_summaries_bucket))
# Clean out actuals_dir first, in case some builders have gone away
# since we last ran.
if os.path.isdir(self._actuals_dir):
shutil.rmtree(self._actuals_dir)
# Get the list of builders we care about.
all_builders = download_actuals.get_builders_list(
summaries_bucket=self._gm_summaries_bucket)
if self._builder_regex_list:
matching_builders = []
for builder in all_builders:
for regex in self._builder_regex_list:
if re.match(regex, builder):
matching_builders.append(builder)
break # go on to the next builder, no need to try more regexes
else:
matching_builders = all_builders
# Download the JSON file for each builder we care about.
#
# TODO(epoger): When this is a large number of builders, we would be
# better off downloading them in parallel!
for builder in matching_builders:
self._gs.download_file(
source_bucket=self._gm_summaries_bucket,
source_path=posixpath.join(builder,
self._json_filename),
dest_path=os.path.join(self._actuals_dir, builder,
self._json_filename),
create_subdirs_if_needed=True)
# We only update the expectations dir if the server was run with a
# nonzero --reload argument; otherwise, we expect the user to maintain
# her own expectations as she sees fit.
#
# Because the Skia repo is hosted using git, and git does not
# support updating a single directory tree, we have to update the entire
# repo checkout.
#
# Because Skia uses depot_tools, we have to update using "gclient sync"
# instead of raw git commands.
#
# TODO(epoger): Fetch latest expectations in some other way.
# Eric points out that our official documentation recommends an
# unmanaged Skia checkout, so "gclient sync" will not bring down updated
# expectations from origin/master-- you'd have to do a "git pull" of
# some sort instead.
# However, the live rebaseline_server at
# http://skia-tree-status.appspot.com/redirect/rebaseline-server (which
# is probably the only user of the --reload flag!) uses a managed
# checkout, so "gclient sync" works in that case.
# Probably the best idea is to avoid all of this nonsense by fetching
# updated expectations into a temp directory, and leaving the rest of
# the checkout alone. This could be done using "git show", or by
# downloading individual expectation JSON files from
# skia.googlesource.com .
if self._reload_seconds:
logging.info(
'Updating expected GM results in %s by syncing Skia repo ...'
% compare_to_expectations.DEFAULT_EXPECTATIONS_DIR)
_run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
self._results = compare_to_expectations.ExpectationComparisons(
image_diff_db=self._image_diff_db,
actuals_root=self._actuals_dir,
diff_base_url=posixpath.join(os.pardir, STATIC_CONTENTS_SUBDIR,
GENERATED_IMAGES_SUBDIR),
builder_regex_list=self._builder_regex_list)
json_dir = os.path.join(PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
GENERATED_JSON_SUBDIR)
if not os.path.isdir(json_dir):
os.makedirs(json_dir)
for config_pair in self._config_pairs:
config_comparisons = compare_configs.ConfigComparisons(
configs=config_pair,
actuals_root=self._actuals_dir,
generated_images_root=os.path.join(
PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
GENERATED_IMAGES_SUBDIR),
diff_base_url=posixpath.join(os.pardir,
GENERATED_IMAGES_SUBDIR),
builder_regex_list=self._builder_regex_list)
for summary_type in _GM_SUMMARY_TYPES:
gm_json.WriteToFile(
config_comparisons.get_packaged_results_of_type(
results_type=summary_type),
os.path.join(
json_dir, '%s-vs-%s_%s.json' %
(config_pair[0], config_pair[1], summary_type)))