def test_validOmissionOnIoError(self, cmd_mock):
def open_impl(_, *args):
# This handles the call to _ClearTriageLinkFile()
if args and args[0] == 'w':
return mock.Mock()
raise IOError('No read today')
cmd_mock.return_value = (1, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file',
None)
m = mock.mock_open()
m.side_effect = open_impl
with mock.patch('__builtin__.open', m, create=True):
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
self.assertEqual(session._comparison_results['name'].triage_link, None)
self.assertNotEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
self.assertIn(
'Failed to read',
session._comparison_results['name'].triage_link_omission_reason)
def test_commandWithUseLuciTrue(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
session.Authenticate(use_luci=True)
self.assertIn('--luci', cmd_mock.call_args[0][0])
def test_commandWithUseLuciFalseNotLocal(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
with self.assertRaises(RuntimeError):
session.Authenticate(use_luci=False)
def test_commandWithLocalPixelTestsFalse(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
session.Compare(None, None)
self.assertNotIn('--dryrun', cmd_mock.call_args[0][0])
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, None)
session.Authenticate()
call_args = cmd_mock.call_args[0][0]
self.assertIn('auth', call_args)
assertArgWith(self, call_args, '--work-dir', working_dir)
def test_commandOutputReturned(self, cmd_mock):
cmd_mock.return_value = (1, 'Something bad :(', None)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, None)
rc, stdout = session.Authenticate()
self.assertEqual(cmd_mock.call_count, 1)
self.assertEqual(rc, 1)
self.assertEqual(stdout, 'Something bad :(')
def test_bypassSkiaGoldFunctionality(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a',
bypass_skia_gold_functionality=True)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
with self.assertRaises(RuntimeError):
session.Diff(None, None, None)
def test_bypassSkiaGoldFunctionality(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a',
bypass_skia_gold_functionality=True)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
rc, _ = session.Compare(None, None)
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
def test_commandOutputReturned(self, cmd_mock):
cmd_mock.return_value = (1, 'Something bad :(', None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
rc, stdout = session.Diff(None, None, None)
self.assertEqual(cmd_mock.call_count, 1)
self.assertEqual(rc, 1)
self.assertEqual(stdout, 'Something bad :(')
def test_shortCircuitAlreadyInitialized(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
session._initialized = True
rc, _ = session.Initialize()
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
def test_authFailure(self, auth_mock, compare_mock, diff_mock):
auth_mock.return_value = (1, 'Auth failed')
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, None)
status, error = session.RunComparison(None, None, None, None)
self.assertEqual(
status, gold_utils.SkiaGoldSession.StatusCodes.AUTH_FAILURE)
self.assertEqual(error, 'Auth failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 0)
self.assertEqual(diff_mock.call_count, 0)
def test_successSetsShortCircuit(self, cmd_mock):
cmd_mock.return_value = (0, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
self.assertFalse(session._authenticated)
rc, _ = session.Authenticate()
self.assertEqual(rc, 0)
self.assertTrue(session._authenticated)
cmd_mock.assert_not_called()
def test_failureDoesNotSetShortCircuit(self, cmd_mock):
cmd_mock.return_value = (1, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
self.assertFalse(session._initialized)
rc, _ = session.Initialize()
self.assertEqual(rc, 1)
self.assertFalse(session._initialized)
cmd_mock.assert_not_called()
def test_noLinkOnSuccess(self, cmd_mock):
cmd_mock.return_value = (0, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp)
rc, _ = session.Compare('name', 'keys_file', 'png_file', None)
self.assertEqual(rc, 0)
self.assertEqual(session._comparison_results['name'].triage_link, None)
self.assertNotEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
def test_commandTryjobArgsMissing(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertNotIn('--issue', call_args)
self.assertNotIn('--patchset', call_args)
self.assertNotIn('--jobid', call_args)
self.assertNotIn('--crs', call_args)
self.assertNotIn('--cis', call_args)
def test_comparisonSuccess(self, auth_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
compare_mock.return_value = (0, None)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
keys_file = os.path.join(working_dir, 'keys.json')
with open(os.path.join(working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = gold_utils.SkiaGoldSession(working_dir, None)
status, _ = session.RunComparison(None, keys_file, None, None)
self.assertEqual(status,
gold_utils.SkiaGoldSession.StatusCodes.SUCCESS)
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_commandTryjobArgs(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None)
session.Initialize()
call_args = cmd_mock.call_args[0][0]
assertArgWith(self, call_args, '--issue', '1')
assertArgWith(self, call_args, '--patchset', '2')
assertArgWith(self, call_args, '--jobid', '3')
assertArgWith(self, call_args, '--crs', 'gerrit')
assertArgWith(self, call_args, '--cis', 'buildbucket')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir,
sgp,
'keys_file',
'corpus',
instance='instance')
session.Compare('name', 'png_file')
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('add', call_args)
assertArgWith(self, call_args, '--test-name', 'name')
assertArgWith(self, call_args, '--png-file', 'png_file')
assertArgWith(self, call_args, '--work-dir', working_dir)
def test_individualLinkOnCi(self, cmd_mock):
cmd_mock.return_value = (1, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp)
m = mock.mock_open(read_data='foobar')
with mock.patch('__builtin__.open', m, create=True):
rc, _ = session.Compare('name', 'keys_file', 'png_file', None)
self.assertEqual(rc, 1)
self.assertNotEqual(session._comparison_results['name'].triage_link,
None)
self.assertEqual(session._comparison_results['name'].triage_link,
'foobar')
self.assertEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
def test_validOmissionOnIoError(self, cmd_mock):
cmd_mock.return_value = (1, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp)
m = mock.mock_open()
m.side_effect = IOError('No read today')
with mock.patch('__builtin__.open', m, create=True):
rc, _ = session.Compare('name', 'keys_file', 'png_file', None)
self.assertEqual(rc, 1)
self.assertEqual(session._comparison_results['name'].triage_link, None)
self.assertNotEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
self.assertIn(
'Failed to read',
session._comparison_results['name'].triage_link_omission_reason)
def test_compareFailureRemote(self, auth_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
args = createSkiaGoldArgs(local_pixel_tests=False)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
keys_file = os.path.join(working_dir, 'keys.json')
with open(os.path.join(working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = gold_utils.SkiaGoldSession(working_dir, sgp)
status, error = session.RunComparison(None, keys_file, None, None)
self.assertEqual(
status, gold_utils.SkiaGoldSession.StatusCodes.
COMPARISON_FAILURE_REMOTE)
self.assertEqual(error, 'Compare failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_clLinkOnTrybot(self, cmd_mock):
cmd_mock.return_value = (1, None, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir, sgp)
rc, _ = session.Compare('name', 'keys_file', 'png_file', None)
self.assertEqual(rc, 1)
self.assertNotEqual(session._comparison_results['name'].triage_link,
None)
self.assertIn('issue=1',
session._comparison_results['name'].triage_link)
self.assertEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
def test_noOutputDirLocal(self, auth_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (0, None)
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
keys_file = os.path.join(working_dir, 'keys.json')
with open(os.path.join(working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = gold_utils.SkiaGoldSession(working_dir, sgp)
status, error = session.RunComparison(None, keys_file, None, None)
self.assertEqual(
status,
gold_utils.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER)
self.assertEqual(error, 'No output manager for local diff images')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_corpusFromJson(self, auth_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
compare_mock.return_value = (0, None)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
keys_file = os.path.join(working_dir, 'keys.json')
with open(os.path.join(working_dir, 'keys.json'), 'w') as f:
json.dump({'source_type': 'foobar'}, f)
session = gold_utils.SkiaGoldSession(working_dir, None,
'SomeCorpus')
status, _ = session.RunComparison(None, keys_file, None, None)
self.assertEqual(status,
gold_utils.SkiaGoldSession.StatusCodes.SUCCESS)
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
compare_mock.assertCalledWith(name=None,
keys_file=keys_file,
png_file=None,
corpus='foobar')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir,
sgp,
instance='instance')
session.Diff('name', 'png_file', 'corpus', None)
call_args = cmd_mock.call_args[0][0]
self.assertIn('diff', call_args)
assertArgWith(self, call_args, '--corpus', 'corpus')
assertArgWith(self, call_args, '--instance', 'instance')
assertArgWith(self, call_args, '--input', 'png_file')
assertArgWith(self, call_args, '--test', 'name')
assertArgWith(self, call_args, '--work-dir', working_dir)
i = call_args.index('--out-dir')
# The output directory should be a subdirectory of the working directory.
self.assertIn(working_dir, call_args[i + 1])
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
session = gold_utils.SkiaGoldSession(working_dir,
sgp,
'keys_file',
'corpus',
instance='instance')
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('init', call_args)
self.assertIn('--passfail', call_args)
assertArgWith(self, call_args, '--instance', 'instance')
assertArgWith(self, call_args, '--corpus', 'corpus')
assertArgWith(self, call_args, '--keys-file', 'keys_file')
assertArgWith(self, call_args, '--work-dir', working_dir)
assertArgWith(self, call_args, '--failure-file',
session._triage_link_file)
assertArgWith(self, call_args, '--commit', 'a')
def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (1, 'Diff failed')
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = gold_utils.SkiaGoldProperties(args)
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
keys_file = os.path.join(working_dir, 'keys.json')
with open(os.path.join(working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = gold_utils.SkiaGoldSession(working_dir, sgp, keys_file,
None)
status, error = session.RunComparison(
None, None, 'Definitely an output manager')
self.assertEqual(
status,
gold_utils.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE)
self.assertEqual(error, 'Diff failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 1)
def _ProcessSkiaGoldRenderTestResults(
self, device, render_tests_device_output_dir, results):
gold_dir = posixpath.join(render_tests_device_output_dir, 'skia_gold')
if not device.FileExists(gold_dir):
return
gold_properties = self._test_instance.skia_gold_properties
with tempfile_ext.NamedTemporaryDirectory() as working_dir:
gold_session = gold_utils.SkiaGoldSession(
working_dir=working_dir, gold_properties=gold_properties)
use_luci = not (gold_properties.local_pixel_tests
or gold_properties.no_luci_auth)
for image_name in device.ListDirectory(gold_dir):
if not image_name.endswith('.png'):
continue
render_name = image_name[:-4]
json_name = render_name + '.json'
device_json_path = posixpath.join(gold_dir, json_name)
if not device.FileExists(device_json_path):
_FailTestIfNecessary(results)
_AppendToLog(
results, 'Unable to find corresponding JSON file for image %s '
'when doing Skia Gold comparison.' % image_name)
continue
host_image_path = os.path.join(working_dir, image_name)
device.PullFile(posixpath.join(gold_dir, image_name), host_image_path)
host_json_path = os.path.join(working_dir, json_name)
device.PullFile(device_json_path, host_json_path)
status, error = gold_session.RunComparison(
name=render_name,
keys_file=host_json_path,
png_file=host_image_path,
output_manager=self._env.output_manager,
use_luci=use_luci)
if not status:
continue
_FailTestIfNecessary(results)
failure_log = (
'Skia Gold reported failure for RenderTest %s. See '
'RENDER_TESTS.md for how to fix this failure.' % render_name)
status_codes = gold_utils.SkiaGoldSession.StatusCodes
if status == status_codes.AUTH_FAILURE:
_AppendToLog(results,
'Gold authentication failed with output %s' % error)
elif status == status_codes.COMPARISON_FAILURE_REMOTE:
triage_link = gold_session.GetTriageLink(render_name)
if not triage_link:
_AppendToLog(
results, 'Failed to get triage link for %s, raw output: %s' %
(render_name, error))
_AppendToLog(
results, 'Reason for no triage link: %s' %
gold_session.GetTriageLinkOmissionReason(render_name))
continue
if gold_properties.IsTryjobRun():
_SetLinkOnResults(results, 'Skia Gold triage link for entire CL',
triage_link)
else:
_SetLinkOnResults(results,
'Skia Gold triage link for %s' % render_name,
triage_link)
_AppendToLog(results, failure_log)
elif status == status_codes.COMPARISON_FAILURE_LOCAL:
given_link = gold_session.GetGivenImageLink(render_name)
closest_link = gold_session.GetClosestImageLink(render_name)
diff_link = gold_session.GetDiffImageLink(render_name)
processed_template_output = _GenerateRenderTestHtml(
render_name, given_link, closest_link, diff_link)
with self._env.output_manager.ArchivedTempfile(
'%s.html' % render_name, 'gold_local_diffs',
output_manager.Datatype.HTML) as html_results:
html_results.write(processed_template_output)
_SetLinkOnResults(results, render_name, html_results.Link())
_AppendToLog(
results,
'See %s link for diff image with closest positive.' % render_name)
elif status == status_codes.LOCAL_DIFF_FAILURE:
_AppendToLog(results,
'Failed to generate diffs from Gold: %s' % error)
else:
logging.error(
'Given unhandled SkiaGoldSession StatusCode %s with error %s',
status, error)
