def test_validOmissionOnIoError(self, cmd_mock): def open_impl(_, *args): # This handles the call to _ClearTriageLinkFile() if args and args[0] == 'w': return mock.Mock() raise IOError('No read today') cmd_mock.return_value = (1, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', None) m = mock.mock_open() m.side_effect = open_impl with mock.patch('__builtin__.open', m, create=True): rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) self.assertEqual(session._comparison_results['name'].triage_link, None) self.assertNotEqual( session._comparison_results['name'].triage_link_omission_reason, None) self.assertIn( 'Failed to read', session._comparison_results['name'].triage_link_omission_reason)
def test_commandWithUseLuciTrue(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) session.Authenticate(use_luci=True) self.assertIn('--luci', cmd_mock.call_args[0][0])
def test_commandWithUseLuciFalseNotLocal(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) with self.assertRaises(RuntimeError): session.Authenticate(use_luci=False)
def test_commandWithLocalPixelTestsFalse(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) session.Compare(None, None) self.assertNotIn('--dryrun', cmd_mock.call_args[0][0])
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None, None) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, None) session.Authenticate() call_args = cmd_mock.call_args[0][0] self.assertIn('auth', call_args) assertArgWith(self, call_args, '--work-dir', working_dir)
def test_commandOutputReturned(self, cmd_mock): cmd_mock.return_value = (1, 'Something bad :(', None) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, None) rc, stdout = session.Authenticate() self.assertEqual(cmd_mock.call_count, 1) self.assertEqual(rc, 1) self.assertEqual(stdout, 'Something bad :(')
def test_bypassSkiaGoldFunctionality(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a', bypass_skia_gold_functionality=True) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) with self.assertRaises(RuntimeError): session.Diff(None, None, None)
def test_bypassSkiaGoldFunctionality(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a', bypass_skia_gold_functionality=True) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) rc, _ = session.Compare(None, None) self.assertEqual(rc, 0) cmd_mock.assert_not_called()
def test_commandOutputReturned(self, cmd_mock): cmd_mock.return_value = (1, 'Something bad :(', None) args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) rc, stdout = session.Diff(None, None, None) self.assertEqual(cmd_mock.call_count, 1) self.assertEqual(rc, 1) self.assertEqual(stdout, 'Something bad :(')
def test_shortCircuitAlreadyInitialized(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) session._initialized = True rc, _ = session.Initialize() self.assertEqual(rc, 0) cmd_mock.assert_not_called()
def test_authFailure(self, auth_mock, compare_mock, diff_mock): auth_mock.return_value = (1, 'Auth failed') with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, None) status, error = session.RunComparison(None, None, None, None) self.assertEqual( status, gold_utils.SkiaGoldSession.StatusCodes.AUTH_FAILURE) self.assertEqual(error, 'Auth failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 0) self.assertEqual(diff_mock.call_count, 0)
def test_successSetsShortCircuit(self, cmd_mock): cmd_mock.return_value = (0, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) self.assertFalse(session._authenticated) rc, _ = session.Authenticate() self.assertEqual(rc, 0) self.assertTrue(session._authenticated) cmd_mock.assert_not_called()
def test_failureDoesNotSetShortCircuit(self, cmd_mock): cmd_mock.return_value = (1, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) self.assertFalse(session._initialized) rc, _ = session.Initialize() self.assertEqual(rc, 1) self.assertFalse(session._initialized) cmd_mock.assert_not_called()
def test_noLinkOnSuccess(self, cmd_mock): cmd_mock.return_value = (0, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp) rc, _ = session.Compare('name', 'keys_file', 'png_file', None) self.assertEqual(rc, 0) self.assertEqual(session._comparison_results['name'].triage_link, None) self.assertNotEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_commandTryjobArgsMissing(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertNotIn('--issue', call_args) self.assertNotIn('--patchset', call_args) self.assertNotIn('--jobid', call_args) self.assertNotIn('--crs', call_args) self.assertNotIn('--cis', call_args)
def test_comparisonSuccess(self, auth_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) compare_mock.return_value = (0, None) with tempfile_ext.NamedTemporaryDirectory() as working_dir: keys_file = os.path.join(working_dir, 'keys.json') with open(os.path.join(working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = gold_utils.SkiaGoldSession(working_dir, None) status, _ = session.RunComparison(None, keys_file, None, None) self.assertEqual(status, gold_utils.SkiaGoldSession.StatusCodes.SUCCESS) self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_commandTryjobArgs(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) session.Initialize() call_args = cmd_mock.call_args[0][0] assertArgWith(self, call_args, '--issue', '1') assertArgWith(self, call_args, '--patchset', '2') assertArgWith(self, call_args, '--jobid', '3') assertArgWith(self, call_args, '--crs', 'gerrit') assertArgWith(self, call_args, '--cis', 'buildbucket')
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', 'corpus', instance='instance') session.Compare('name', 'png_file') call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('add', call_args) assertArgWith(self, call_args, '--test-name', 'name') assertArgWith(self, call_args, '--png-file', 'png_file') assertArgWith(self, call_args, '--work-dir', working_dir)
def test_individualLinkOnCi(self, cmd_mock): cmd_mock.return_value = (1, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp) m = mock.mock_open(read_data='foobar') with mock.patch('__builtin__.open', m, create=True): rc, _ = session.Compare('name', 'keys_file', 'png_file', None) self.assertEqual(rc, 1) self.assertNotEqual(session._comparison_results['name'].triage_link, None) self.assertEqual(session._comparison_results['name'].triage_link, 'foobar') self.assertEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_validOmissionOnIoError(self, cmd_mock): cmd_mock.return_value = (1, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp) m = mock.mock_open() m.side_effect = IOError('No read today') with mock.patch('__builtin__.open', m, create=True): rc, _ = session.Compare('name', 'keys_file', 'png_file', None) self.assertEqual(rc, 1) self.assertEqual(session._comparison_results['name'].triage_link, None) self.assertNotEqual( session._comparison_results['name'].triage_link_omission_reason, None) self.assertIn( 'Failed to read', session._comparison_results['name'].triage_link_omission_reason)
def test_compareFailureRemote(self, auth_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') args = createSkiaGoldArgs(local_pixel_tests=False) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: keys_file = os.path.join(working_dir, 'keys.json') with open(os.path.join(working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = gold_utils.SkiaGoldSession(working_dir, sgp) status, error = session.RunComparison(None, keys_file, None, None) self.assertEqual( status, gold_utils.SkiaGoldSession.StatusCodes. COMPARISON_FAILURE_REMOTE) self.assertEqual(error, 'Compare failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_clLinkOnTrybot(self, cmd_mock): cmd_mock.return_value = (1, None, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp) rc, _ = session.Compare('name', 'keys_file', 'png_file', None) self.assertEqual(rc, 1) self.assertNotEqual(session._comparison_results['name'].triage_link, None) self.assertIn('issue=1', session._comparison_results['name'].triage_link) self.assertEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_noOutputDirLocal(self, auth_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=True) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: keys_file = os.path.join(working_dir, 'keys.json') with open(os.path.join(working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = gold_utils.SkiaGoldSession(working_dir, sgp) status, error = session.RunComparison(None, keys_file, None, None) self.assertEqual( status, gold_utils.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) self.assertEqual(error, 'No output manager for local diff images') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_corpusFromJson(self, auth_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) compare_mock.return_value = (0, None) with tempfile_ext.NamedTemporaryDirectory() as working_dir: keys_file = os.path.join(working_dir, 'keys.json') with open(os.path.join(working_dir, 'keys.json'), 'w') as f: json.dump({'source_type': 'foobar'}, f) session = gold_utils.SkiaGoldSession(working_dir, None, 'SomeCorpus') status, _ = session.RunComparison(None, keys_file, None, None) self.assertEqual(status, gold_utils.SkiaGoldSession.StatusCodes.SUCCESS) self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0) compare_mock.assertCalledWith(name=None, keys_file=keys_file, png_file=None, corpus='foobar')
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, instance='instance') session.Diff('name', 'png_file', 'corpus', None) call_args = cmd_mock.call_args[0][0] self.assertIn('diff', call_args) assertArgWith(self, call_args, '--corpus', 'corpus') assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--input', 'png_file') assertArgWith(self, call_args, '--test', 'name') assertArgWith(self, call_args, '--work-dir', working_dir) i = call_args.index('--out-dir') # The output directory should be a subdirectory of the working directory. self.assertIn(working_dir, call_args[i + 1])
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None, None) args = createSkiaGoldArgs(git_revision='a') sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', 'corpus', instance='instance') session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('init', call_args) self.assertIn('--passfail', call_args) assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--corpus', 'corpus') assertArgWith(self, call_args, '--keys-file', 'keys_file') assertArgWith(self, call_args, '--work-dir', working_dir) assertArgWith(self, call_args, '--failure-file', session._triage_link_file) assertArgWith(self, call_args, '--commit', 'a')
def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (1, 'Diff failed') args = createSkiaGoldArgs(local_pixel_tests=True) sgp = gold_utils.SkiaGoldProperties(args) with tempfile_ext.NamedTemporaryDirectory() as working_dir: keys_file = os.path.join(working_dir, 'keys.json') with open(os.path.join(working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = gold_utils.SkiaGoldSession(working_dir, sgp, keys_file, None) status, error = session.RunComparison( None, None, 'Definitely an output manager') self.assertEqual( status, gold_utils.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) self.assertEqual(error, 'Diff failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 1)
def _ProcessSkiaGoldRenderTestResults( self, device, render_tests_device_output_dir, results): gold_dir = posixpath.join(render_tests_device_output_dir, 'skia_gold') if not device.FileExists(gold_dir): return gold_properties = self._test_instance.skia_gold_properties with tempfile_ext.NamedTemporaryDirectory() as working_dir: gold_session = gold_utils.SkiaGoldSession( working_dir=working_dir, gold_properties=gold_properties) use_luci = not (gold_properties.local_pixel_tests or gold_properties.no_luci_auth) for image_name in device.ListDirectory(gold_dir): if not image_name.endswith('.png'): continue render_name = image_name[:-4] json_name = render_name + '.json' device_json_path = posixpath.join(gold_dir, json_name) if not device.FileExists(device_json_path): _FailTestIfNecessary(results) _AppendToLog( results, 'Unable to find corresponding JSON file for image %s ' 'when doing Skia Gold comparison.' % image_name) continue host_image_path = os.path.join(working_dir, image_name) device.PullFile(posixpath.join(gold_dir, image_name), host_image_path) host_json_path = os.path.join(working_dir, json_name) device.PullFile(device_json_path, host_json_path) status, error = gold_session.RunComparison( name=render_name, keys_file=host_json_path, png_file=host_image_path, output_manager=self._env.output_manager, use_luci=use_luci) if not status: continue _FailTestIfNecessary(results) failure_log = ( 'Skia Gold reported failure for RenderTest %s. See ' 'RENDER_TESTS.md for how to fix this failure.' % render_name) status_codes = gold_utils.SkiaGoldSession.StatusCodes if status == status_codes.AUTH_FAILURE: _AppendToLog(results, 'Gold authentication failed with output %s' % error) elif status == status_codes.COMPARISON_FAILURE_REMOTE: triage_link = gold_session.GetTriageLink(render_name) if not triage_link: _AppendToLog( results, 'Failed to get triage link for %s, raw output: %s' % (render_name, error)) _AppendToLog( results, 'Reason for no triage link: %s' % gold_session.GetTriageLinkOmissionReason(render_name)) continue if gold_properties.IsTryjobRun(): _SetLinkOnResults(results, 'Skia Gold triage link for entire CL', triage_link) else: _SetLinkOnResults(results, 'Skia Gold triage link for %s' % render_name, triage_link) _AppendToLog(results, failure_log) elif status == status_codes.COMPARISON_FAILURE_LOCAL: given_link = gold_session.GetGivenImageLink(render_name) closest_link = gold_session.GetClosestImageLink(render_name) diff_link = gold_session.GetDiffImageLink(render_name) processed_template_output = _GenerateRenderTestHtml( render_name, given_link, closest_link, diff_link) with self._env.output_manager.ArchivedTempfile( '%s.html' % render_name, 'gold_local_diffs', output_manager.Datatype.HTML) as html_results: html_results.write(processed_template_output) _SetLinkOnResults(results, render_name, html_results.Link()) _AppendToLog( results, 'See %s link for diff image with closest positive.' % render_name) elif status == status_codes.LOCAL_DIFF_FAILURE: _AppendToLog(results, 'Failed to generate diffs from Gold: %s' % error) else: logging.error( 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status, error)