def test_noLinkOnSuccess(self, cmd_mock): cmd_mock.return_value = (0, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, None) rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 0) comparison_result = session._comparison_results['name'] self.assertEqual(comparison_result.public_triage_link, None) self.assertEqual(comparison_result.internal_triage_link, None) self.assertNotEqual(comparison_result.triage_link_omission_reason, None)
def test_initFailure(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (1, 'Init failed') session = skia_gold_session.SkiaGoldSession(self._working_dir, None, None, None, None) status, error = session.RunComparison(None, None, None) self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes.INIT_FAILURE) self.assertEqual(error, 'Init failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 0) self.assertEqual(diff_mock.call_count, 0)
def test_commandTryjobArgsMissing(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, None, None) session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertNotIn('--issue', call_args) self.assertNotIn('--patchset', call_args) self.assertNotIn('--jobid', call_args) self.assertNotIn('--crs', call_args) self.assertNotIn('--cis', call_args)
def test_authFailure(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (1, 'Auth failed') sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, error = session.RunComparison(None, None, None) self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes.AUTH_FAILURE) self.assertEqual(error, 'Auth failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 0) self.assertEqual(compare_mock.call_count, 0) self.assertEqual(diff_mock.call_count, 0)
def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (0, None) session = skia_gold_session.SkiaGoldSession(self._working_dir, None, self._json_keys, None, None) status, _ = session.RunComparison(None, None, None) self.assertEqual(status, skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_commandTryjobArgs(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, None, None) session.Initialize() call_args = cmd_mock.call_args[0][0] assertArgWith(self, call_args, '--issue', '1') assertArgWith(self, call_args, '--patchset', '2') assertArgWith(self, call_args, '--jobid', '3') assertArgWith(self, call_args, '--crs', 'gerrit') assertArgWith(self, call_args, '--cis', 'buildbucket')
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, 'corpus', instance='instance') session.Compare('name', 'png_file') call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('add', call_args) assertArgWith(self, call_args, '--test-name', 'name') assertArgWith(self, call_args, '--png-file', 'png_file') assertArgWith(self, call_args, '--work-dir', self._working_dir)
def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (0, None) keys_file = os.path.join(self._working_dir, 'keys.json') with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = skia_gold_session.SkiaGoldSession(self._working_dir, None, keys_file, None, None) status, _ = session.RunComparison(None, None, None) self.assertEqual(status, skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') args = createSkiaGoldArgs(local_pixel_tests=False) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, error = session.RunComparison(None, None, None) self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes. COMPARISON_FAILURE_REMOTE) self.assertEqual(error, 'Compare failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_noOutputManagerLocal(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, error = session.RunComparison(None, None, None) self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) self.assertEqual(error, 'No output manager for local diff images') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_validOmissionOnMissingLink(self, cmd_mock): args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, None) def WriteTriageLinkFile(_): with open(session._triage_link_file, 'w'): pass return (1, None) cmd_mock.side_effect = WriteTriageLinkFile rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) comparison_result = session._comparison_results['name'] self.assertEqual(comparison_result.public_triage_link, None) self.assertEqual(comparison_result.internal_triage_link, None) self.assertIn('Gold did not provide a triage link', comparison_result.triage_link_omission_reason)
def test_validOmissionOnIoError(self, cmd_mock): cmd_mock.return_value = (1, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, None) def DeleteTriageLinkFile(_): os.remove(session._triage_link_file) return (1, None) cmd_mock.side_effect = DeleteTriageLinkFile rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) comparison_result = session._comparison_results['name'] self.assertEqual(comparison_result.public_triage_link, None) self.assertEqual(comparison_result.internal_triage_link, None) self.assertNotEqual(comparison_result.triage_link_omission_reason, None) self.assertIn('Failed to read', comparison_result.triage_link_omission_reason)
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', 'corpus', instance='instance') session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('init', call_args) self.assertIn('--passfail', call_args) assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--corpus', 'corpus') assertArgWith(self, call_args, '--keys-file', 'keys_file') assertArgWith(self, call_args, '--work-dir', self._working_dir) assertArgWith(self, call_args, '--failure-file', session._triage_link_file) assertArgWith(self, call_args, '--commit', 'a')
def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (1, 'Diff failed') args = createSkiaGoldArgs(local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, error = session.RunComparison(None, None, 'Definitely an output manager') self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) self.assertEqual(error, 'Diff failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 1)
def test_clLinkOnTrybot(self, cmd_mock): cmd_mock.return_value = (1, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, instance='instance') rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) self.assertNotEqual(session._comparison_results['name'].triage_link, None) self.assertEqual(session._comparison_results['name'].triage_link, 'https://instance-gold.skia.org/cl/gerrit/1') self.assertEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_individualLinkOnCi(self, cmd_mock): args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, None) def WriteTriageLinkFile(_): with open(session._triage_link_file, 'w') as f: f.write('foobar') return (1, None) cmd_mock.side_effect = WriteTriageLinkFile rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) self.assertNotEqual(session._comparison_results['name'].triage_link, None) self.assertEqual(session._comparison_results['name'].triage_link, 'foobar') self.assertEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, 'corpus', instance='instance') session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('init', call_args) self.assertIn('--passfail', call_args) assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--corpus', 'corpus') # The keys file should have been copied to the working directory. assertArgWith(self, call_args, '--keys-file', os.path.join(self._working_dir, 'gold_keys.json')) assertArgWith(self, call_args, '--work-dir', self._working_dir) assertArgWith(self, call_args, '--failure-file', session._triage_link_file) assertArgWith(self, call_args, '--commit', 'a')
def test_compareForceDryrun(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (0, None) diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=False) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, _ = session.RunComparison(None, None, None, force_dryrun=True) self.assertEqual(status, skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0) compare_mock.assert_called_with(name=None, png_file=mock.ANY, inexact_matching_args=None, optional_keys=None, force_dryrun=True)
def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) keys_file = os.path.join(self._working_dir, 'keys.json') with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, keys_file, None, None) status, error = session.RunComparison(None, None, 'Definitely an output manager') self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes. COMPARISON_FAILURE_LOCAL) self.assertEqual(error, 'Compare failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 1)
def test_clLinkOnTrybot(self, cmd_mock): cmd_mock.return_value = (1, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, instance='instance') rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) comparison_result = session._comparison_results['name'] self.assertNotEqual(comparison_result.public_triage_link, None) self.assertNotEqual(comparison_result.internal_triage_link, None) internal_link = 'https://instance-gold.skia.org/cl/gerrit/1' public_link = 'https://instance-public-gold.skia.org/cl/gerrit/1' self.assertEqual(comparison_result.internal_triage_link, internal_link) self.assertEqual(comparison_result.public_triage_link, public_link) self.assertEqual(comparison_result.triage_link_omission_reason, None) self.assertEqual(session.GetTriageLinks('name'), (public_link, internal_link))