def _UploadTestResultToSkiaGold(self, image_name, screenshot, page):
"""Compares the given image using Skia Gold and uploads the result.
No uploading is done if the test is being run in local run mode. Compares
the given screenshot to baselines provided by Gold, raising an Exception if
a match is not found.
Args:
image_name: the name of the image being checked.
screenshot: the image being checked as a Telemetry Bitmap.
page: the GPU PixelTestPage object for the test.
"""
# Write screenshot to PNG file on local disk.
png_temp_file = tempfile.NamedTemporaryFile(
suffix='.png', dir=self._skia_gold_temp_dir).name
image_util.WritePngFile(screenshot, png_temp_file)
gpu_keys = self.GetGoldJsonKeys(page)
gold_session = self.GetSkiaGoldSessionManager().GetSkiaGoldSession(
gpu_keys)
gold_properties = self.GetSkiaGoldProperties()
use_luci = not (gold_properties.local_pixel_tests
or gold_properties.no_luci_auth)
inexact_matching_args = page.matching_algorithm.GetCmdline()
status, error = gold_session.RunComparison(
name=image_name,
png_file=png_temp_file,
inexact_matching_args=inexact_matching_args,
use_luci=use_luci)
if not status:
return
status_codes = gpu_skia_gold_session.GpuSkiaGoldSession.StatusCodes
if status == status_codes.AUTH_FAILURE:
logging.error('Gold authentication failed with output %s', error)
elif status == status_codes.INIT_FAILURE:
logging.error('Gold initialization failed with output %s', error)
elif status == status_codes.COMPARISON_FAILURE_REMOTE:
# We currently don't have an internal instance + public mirror like the
# general Chrome Gold instance, so just report the "internal" link, which
# points to the correct instance.
_, triage_link = gold_session.GetTriageLinks(image_name)
if not triage_link:
logging.error(
'Failed to get triage link for %s, raw output: %s',
image_name, error)
logging.error(
'Reason for no triage link: %s',
gold_session.GetTriageLinkOmissionReason(image_name))
elif gold_properties.IsTryjobRun():
self.artifacts.CreateLink('triage_link_for_entire_cl',
triage_link)
else:
self.artifacts.CreateLink('gold_triage_link', triage_link)
elif status == status_codes.COMPARISON_FAILURE_LOCAL:
logging.error('Local comparison failed. Local diff files:')
_OutputLocalDiffFiles(gold_session, image_name)
elif status == status_codes.LOCAL_DIFF_FAILURE:
logging.error(
'Local comparison failed and an error occurred during diff '
'generation: %s', error)
# There might be some files, so try outputting them.
logging.error('Local diff files:')
_OutputLocalDiffFiles(gold_session, image_name)
else:
logging.error(
'Given unhandled SkiaGoldSession StatusCode %s with error %s',
status, error)
if self._ShouldReportGoldFailure(page):
raise Exception('goldctl command failed, see above for details')
def TakeScreenshot(self, file_path):
if not self.CanTakeScreenshot():
raise NotImplementedError
img = image_util.FromBase64Png(self.screenshot_png_data)
image_util.WritePngFile(img, file_path)
return True
def _WriteImage(self, image_path, png_image):
output_dir = os.path.dirname(image_path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
image_util.WritePngFile(png_image, image_path)
def _UploadTestResultToSkiaGold(self, image_name, screenshot, page):
"""Compares the given image using Skia Gold and uploads the result.
No uploading is done if the test is being run in local run mode. Compares
the given screenshot to baselines provided by Gold, raising an Exception if
a match is not found.
Args:
image_name: the name of the image being checked.
screenshot: the image being checked as a Telemetry Bitmap.
page: the GPU PixelTestPage object for the test.
"""
# Write screenshot to PNG file on local disk.
png_temp_file = tempfile.NamedTemporaryFile(
suffix='.png', dir=self._skia_gold_temp_dir).name
image_util.WritePngFile(screenshot, png_temp_file)
gpu_keys = self.GetGoldJsonKeys(page)
gold_session = self.GetSkiaGoldSessionManager().GetSkiaGoldSession(
gpu_keys, corpus=SKIA_GOLD_CORPUS)
gold_properties = self.GetSkiaGoldProperties()
use_luci = not (gold_properties.local_pixel_tests
or gold_properties.no_luci_auth)
inexact_matching_args = page.matching_algorithm.GetCmdline()
# TODO(skbug.com/12149): Remove this once Gold stops clobbering earlier
# results when running retry steps.
force_dryrun = False
# "Retry without patch" steps automatically pass in a test filter, which
# should be the only time these tests are run with a test filter on trybots.
if (gold_properties.IsTryjobRun()
and self.GetParsedCommandLineOptions().has_test_filter):
force_dryrun = True
status, error = gold_session.RunComparison(
name=image_name,
png_file=png_temp_file,
inexact_matching_args=inexact_matching_args,
use_luci=use_luci,
force_dryrun=force_dryrun)
if not status:
return
status_codes =\
self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes
if status == status_codes.AUTH_FAILURE:
logging.error('Gold authentication failed with output %s', error)
elif status == status_codes.INIT_FAILURE:
logging.error('Gold initialization failed with output %s', error)
elif status == status_codes.COMPARISON_FAILURE_REMOTE:
# We currently don't have an internal instance + public mirror like the
# general Chrome Gold instance, so just report the "internal" link, which
# points to the correct instance.
_, triage_link = gold_session.GetTriageLinks(image_name)
if not triage_link:
logging.error('Failed to get triage link for %s, raw output: %s',
image_name, error)
logging.error('Reason for no triage link: %s',
gold_session.GetTriageLinkOmissionReason(image_name))
elif gold_properties.IsTryjobRun():
self.artifacts.CreateLink('triage_link_for_entire_cl', triage_link)
else:
self.artifacts.CreateLink('gold_triage_link', triage_link)
elif status == status_codes.COMPARISON_FAILURE_LOCAL:
logging.error('Local comparison failed. Local diff files:')
_OutputLocalDiffFiles(gold_session, image_name)
elif status == status_codes.LOCAL_DIFF_FAILURE:
logging.error(
'Local comparison failed and an error occurred during diff '
'generation: %s', error)
# There might be some files, so try outputting them.
logging.error('Local diff files:')
_OutputLocalDiffFiles(gold_session, image_name)
else:
logging.error(
'Given unhandled SkiaGoldSession StatusCode %s with error %s', status,
error)
if self._ShouldReportGoldFailure(page):
raise Exception(
'goldctl command returned non-zero exit code, see above for details. '
'This probably just means that the test produced an image that has '
'not been triaged as positive.')
def _UploadTestResultToSkiaGold(self,
image_name,
screenshot,
page,
build_id_args=None):
"""Compares the given image using Skia Gold and uploads the result.
No uploading is done if the test is being run in local run mode. Compares
the given screenshot to baselines provided by Gold, raising an Exception if
a match is not found.
Args:
image_name: the name of the image being checked.
screenshot: the image being checked as a Telemetry Bitmap.
page: the GPU PixelTestPage object for the test.
build_id_args: a list of build-identifying flags and values.
"""
if self.GetParsedCommandLineOptions().bypass_skia_gold_functionality:
logging.warning('Not actually comparing with Gold due to '
'--bypass-skia-gold-functionality being present.')
return
if not isinstance(build_id_args,
list) or '--commit' not in build_id_args:
raise Exception(
'Requires build args to be specified, including --commit')
# Write screenshot to PNG file on local disk.
png_temp_file = tempfile.NamedTemporaryFile(
suffix='.png', dir=self._skia_gold_temp_dir).name
image_util.WritePngFile(screenshot, png_temp_file)
gpu_keys = self.GetGoldJsonKeys(page)
json_temp_file = tempfile.NamedTemporaryFile(
suffix='.json', dir=self._skia_gold_temp_dir).name
failure_file = tempfile.NamedTemporaryFile(
suffix='.txt', dir=self._skia_gold_temp_dir).name
with open(json_temp_file, 'w+') as f:
json.dump(gpu_keys, f)
# Figure out any extra args we need to pass to goldctl.
extra_imgtest_args = []
extra_auth_args = []
parsed_options = self.GetParsedCommandLineOptions()
if self._IsLocalRun():
extra_imgtest_args.append('--dryrun')
elif not parsed_options.no_luci_auth:
extra_auth_args = ['--luci']
# Run goldctl for a result.
try:
subprocess.check_output(
[GOLDCTL_BIN, 'auth', '--work-dir', self._skia_gold_temp_dir] +
extra_auth_args,
stderr=subprocess.STDOUT)
algorithm_args = page.matching_algorithm.GetCmdline()
if algorithm_args:
logging.info('Using non-exact matching algorithm %s for %s',
page.matching_algorithm.Name(), image_name)
# yapf: disable
cmd = ([
GOLDCTL_BIN,
'imgtest', 'add',
'--passfail',
'--test-name', image_name,
'--instance', SKIA_GOLD_INSTANCE,
'--keys-file', json_temp_file,
'--png-file', png_temp_file,
'--work-dir', self._skia_gold_temp_dir,
'--failure-file', failure_file
] + build_id_args + extra_imgtest_args + algorithm_args)
# yapf: enable
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
# TODO(skbug.com/10245): Remove this once the issue with auto-triaging
# inexactly matched images is fixed.
if 'VP9_YUY2' in image_name:
logging.error(output)
except CalledProcessError as e:
# We don't want to bother printing out triage links for local runs.
# Instead, we print out local filepaths for debugging. However, we want
# these to be at the bottom of the output so they're easier to find, so
# that is handled later.
if self._IsLocalRun():
pass
# The triage link for the image is output to the failure file, so report
# that if it's available so it shows up in Milo. If for whatever reason
# the file is not present or malformed, the triage link will still be
# present in the stdout of the goldctl command.
# If we're running on a trybot, instead generate a link to all results
# for the CL so that the user can visit a single page instead of
# clicking on multiple links on potentially multiple bots.
elif parsed_options.gerrit_issue:
cl_images = ('https://%s-gold.skia.org/search?'
'issue=%s&new_clstore=true' %
(SKIA_GOLD_INSTANCE, parsed_options.gerrit_issue))
self.artifacts.CreateLink('triage_link_for_entire_cl',
cl_images)
else:
try:
with open(failure_file, 'r') as ff:
self.artifacts.CreateLink('gold_triage_link',
ff.read())
except Exception:
logging.error(
'Failed to read contents of goldctl failure file')
logging.error('goldctl failed with output: %s', e.output)
if self._IsLocalRun():
# Intentionally not cleaned up so the user can look at its contents.
diff_dir = tempfile.mkdtemp()
# yapf: disable
cmd = [
GOLDCTL_BIN,
'diff',
'--corpus', SKIA_GOLD_CORPUS,
'--instance', SKIA_GOLD_INSTANCE,
'--input', png_temp_file,
'--test', image_name,
'--work-dir', self._skia_gold_temp_dir,
'--out-dir', diff_dir,
]
# yapf: enable
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except CalledProcessError as e:
logging.error('Failed to generate diffs from Gold: %s', e)
# The directory should contain "input-<hash>.png", "closest-<hash>.png",
# and "diff.png".
for f in os.listdir(diff_dir):
filepath = os.path.join(diff_dir, f)
if f.startswith("input-"):
logging.error("Image produced by %s: file://%s",
image_name, filepath)
elif f.startswith("closest-"):
logging.error("Closest image for %s: file://%s",
image_name, filepath)
elif f == "diff.png":
logging.error("Diff image for %s: file://%s",
image_name, filepath)
if self._ShouldReportGoldFailure(page):
raise Exception(
'goldctl command failed, see above for details')
def _UploadTestResultToSkiaGold(self,
image_name,
screenshot,
tab,
page,
build_id_args=None):
"""Compares the given image using Skia Gold and uploads the result.
No uploading is done if the test is being run in local run mode. Compares
the given screenshot to baselines provided by Gold, raising an Exception if
a match is not found.
Args:
image_name: the name of the image being checked.
screenshot: the image being checked as a Telemetry Bitmap.
tab: the Telemetry Tab object that the test was run in.
page: the GPU PixelTestPage object for the test.
build_id_args: a list of build-identifying flags and values.
"""
if not isinstance(build_id_args,
list) or '--commit' not in build_id_args:
raise Exception(
'Requires build args to be specified, including --commit')
# Write screenshot to PNG file on local disk.
png_temp_file = tempfile.NamedTemporaryFile(
suffix='.png', dir=self._skia_gold_temp_dir).name
image_util.WritePngFile(screenshot, png_temp_file)
# Get all information that goldctl will need.
img_params = self.GetImageParameters(tab, page)
# All values need to be strings, otherwise goldctl fails.
gpu_keys = {
'vendor_id': self.ToHexOrNone(img_params.vendor_id),
'device_id': self.ToHexOrNone(img_params.device_id),
'vendor_string': str(img_params.vendor_string),
'device_string': str(img_params.device_string),
'msaa': str(img_params.msaa),
'model_name': str(img_params.model_name),
}
json_temp_file = tempfile.NamedTemporaryFile(
suffix='.json', dir=self._skia_gold_temp_dir).name
failure_file = tempfile.NamedTemporaryFile(
suffix='.txt', dir=self._skia_gold_temp_dir).name
with open(json_temp_file, 'w+') as f:
json.dump(gpu_keys, f)
# Figure out any extra args we need to pass to goldctl.
extra_imgtest_args = []
extra_auth_args = []
parsed_options = self.GetParsedCommandLineOptions()
if parsed_options.local_run:
extra_imgtest_args.append('--dryrun')
elif not parsed_options.no_luci_auth:
extra_auth_args = ['--luci']
# Run goldctl for a result.
try:
subprocess.check_output(
[goldctl_bin, 'auth', '--work-dir', self._skia_gold_temp_dir] +
extra_auth_args,
stderr=subprocess.STDOUT)
cmd = ([
goldctl_bin, 'imgtest', 'add', '--passfail', '--test-name',
image_name, '--instance', SKIA_GOLD_INSTANCE, '--keys-file',
json_temp_file, '--png-file', png_temp_file, '--work-dir',
self._skia_gold_temp_dir, '--failure-file', failure_file
] + build_id_args + extra_imgtest_args)
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except CalledProcessError as e:
try:
# The triage link for the image is output to the failure file, so report
# that if it's available so it shows up in Milo. If for whatever reason
# the file is not present or malformed, the triage link will still be
# present in the stdout of the goldctl command.
with open(failure_file, 'r') as ff:
self.artifacts.CreateLink('gold_triage_link', ff.read())
except Exception:
logging.error(
'Failed to read contents of goldctl failure file')
logging.error('goldctl failed with output: %s', e.output)
if parsed_options.local_run:
logging.error('Image produced by %s: file://%s', image_name,
png_temp_file)
gold_images = ('https://%s-gold.skia.org/search?'
'match=name&metric=combined&pos=true&'
'query=name%%3D%s&unt=false' %
(SKIA_GOLD_INSTANCE, image_name))
logging.error('Approved images for %s in Gold: %s', image_name,
gold_images)
if not parsed_options.no_skia_gold_failure:
raise Exception('goldctl command failed')
