def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, error = session.RunComparison(None, None, 'Definitely an output manager') self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes. COMPARISON_FAILURE_LOCAL) self.assertEqual(error, 'Compare failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 1)
def test_clLinkOnTrybot(self, cmd_mock): cmd_mock.return_value = (1, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, instance='instance') rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) self.assertNotEqual(session._comparison_results['name'].triage_link, None) self.assertEqual(session._comparison_results['name'].triage_link, 'https://instance-gold.skia.org/cl/gerrit/1') self.assertEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', 'corpus', instance='instance') session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('init', call_args) self.assertIn('--passfail', call_args) assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--corpus', 'corpus') assertArgWith(self, call_args, '--keys-file', 'keys_file') assertArgWith(self, call_args, '--work-dir', self._working_dir) assertArgWith(self, call_args, '--failure-file', session._triage_link_file) assertArgWith(self, call_args, '--commit', 'a')
def test_noOutputManagerLocal(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) keys_file = os.path.join(self._working_dir, 'keys.json') with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, keys_file, None, None) status, error = session.RunComparison(None, None, None) self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) self.assertEqual(error, 'No output manager for local diff images') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') args = createSkiaGoldArgs(local_pixel_tests=False) sgp = skia_gold_properties.SkiaGoldProperties(args) keys_file = os.path.join(self._working_dir, 'keys.json') with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, keys_file, None, None) status, error = session.RunComparison(None, None, None) self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes. COMPARISON_FAILURE_REMOTE) self.assertEqual(error, 'Compare failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0)
def test_individualLinkOnCi(self, cmd_mock): args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, None) def WriteTriageLinkFile(_): with open(session._triage_link_file, 'w') as f: f.write('foobar') return (1, None) cmd_mock.side_effect = WriteTriageLinkFile rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) self.assertNotEqual(session._comparison_results['name'].triage_link, None) self.assertEqual(session._comparison_results['name'].triage_link, 'foobar') self.assertEqual( session._comparison_results['name'].triage_link_omission_reason, None)
def test_validOmissionOnIoError(self, cmd_mock): cmd_mock.return_value = (1, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 'keys_file', None, None) def DeleteTriageLinkFile(_): os.remove(session._triage_link_file) return (1, None) cmd_mock.side_effect = DeleteTriageLinkFile rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) comparison_result = session._comparison_results['name'] self.assertEqual(comparison_result.public_triage_link, None) self.assertEqual(comparison_result.internal_triage_link, None) self.assertNotEqual(comparison_result.triage_link_omission_reason, None) self.assertIn('Failed to read', comparison_result.triage_link_omission_reason)
def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (1, 'Compare failed') diff_mock.return_value = (1, 'Diff failed') args = createSkiaGoldArgs(local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) keys_file = os.path.join(self._working_dir, 'keys.json') with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: json.dump({}, f) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, keys_file, None, None) status, error = session.RunComparison(None, None, 'Definitely an output manager') self.assertEqual( status, skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) self.assertEqual(error, 'Diff failed') self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 1)
def test_commandCommonArgs(self, cmd_mock): cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, 'corpus', instance='instance') session.Initialize() call_args = cmd_mock.call_args[0][0] self.assertIn('imgtest', call_args) self.assertIn('init', call_args) self.assertIn('--passfail', call_args) assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--corpus', 'corpus') # The keys file should have been copied to the working directory. assertArgWith(self, call_args, '--keys-file', os.path.join(self._working_dir, 'gold_keys.json')) assertArgWith(self, call_args, '--work-dir', self._working_dir) assertArgWith(self, call_args, '--failure-file', session._triage_link_file) assertArgWith(self, call_args, '--commit', 'a')
def test_outputManagerNotNeeded(self): args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) sgp = skia_gold_properties.SkiaGoldProperties(args) session = omsgs.OutputManagerlessSkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) input_filepath = os.path.join(self._working_dir, 'input-inputhash.png') with open(input_filepath, 'w') as f: f.write('') closest_filepath = os.path.join(self._working_dir, 'closest-closesthash.png') with open(closest_filepath, 'w') as f: f.write('') diff_filepath = os.path.join(self._working_dir, 'diff.png') with open(diff_filepath, 'w') as f: f.write('') session._StoreDiffLinks('foo', None, self._working_dir) self.assertEqual(session.GetGivenImageLink('foo'), 'file://' + input_filepath) self.assertEqual(session.GetClosestImageLink('foo'), 'file://' + closest_filepath) self.assertEqual(session.GetDiffImageLink('foo'), 'file://' + diff_filepath)
def test_compareForceDryrun(self, auth_mock, init_mock, compare_mock, diff_mock): auth_mock.return_value = (0, None) init_mock.return_value = (0, None) compare_mock.return_value = (0, None) diff_mock.return_value = (0, None) args = createSkiaGoldArgs(local_pixel_tests=False) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, None) status, _ = session.RunComparison(None, None, None, force_dryrun=True) self.assertEqual(status, skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) self.assertEqual(auth_mock.call_count, 1) self.assertEqual(init_mock.call_count, 1) self.assertEqual(compare_mock.call_count, 1) self.assertEqual(diff_mock.call_count, 0) compare_mock.assert_called_with(name=None, png_file=mock.ANY, inexact_matching_args=None, optional_keys=None, force_dryrun=True)
def test_clLinkOnTrybot(self, cmd_mock): cmd_mock.return_value = (1, None) args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = skia_gold_properties.SkiaGoldProperties(args) session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, self._json_keys, None, instance='instance') rc, _ = session.Compare('name', 'png_file') self.assertEqual(rc, 1) comparison_result = session._comparison_results['name'] self.assertNotEqual(comparison_result.public_triage_link, None) self.assertNotEqual(comparison_result.internal_triage_link, None) internal_link = 'https://instance-gold.skia.org/cl/gerrit/1' public_link = 'https://instance-public-gold.skia.org/cl/gerrit/1' self.assertEqual(comparison_result.internal_triage_link, internal_link) self.assertEqual(comparison_result.public_triage_link, public_link) self.assertEqual(comparison_result.triage_link_omission_reason, None) self.assertEqual(session.GetTriageLinks('name'), (public_link, internal_link))
def test_initializeSkiaGoldAttributes_explicitCrs(self): args = createSkiaGoldArgs(code_review_system='foo') sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {'code_review_system': 'foo'})
def test_initializeSkiaGoldAttributes_unsetLocal(self): args = createSkiaGoldArgs() sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {})
def testLocalPixelTests_determineFalse(self): args = createSkiaGoldArgs() sgp = skia_gold_properties.SkiaGoldProperties(args) with mock.patch.dict(os.environ, {'SWARMING_SERVER': ''}, clear=True): self.assertFalse(sgp.local_pixel_tests)
def test_initializeSkiaGoldAttributes_tryjobArgsIgnoredWithoutRevision(self): args = createSkiaGoldArgs(gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {})
def testGetGitRevision_revisionSet(self): args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) self.assertEqual(sgp.git_revision, 'a')
def testIsTryjobRun_noIssue(self): args = createSkiaGoldArgs() sgp = skia_gold_properties.SkiaGoldProperties(args) self.assertFalse(sgp.IsTryjobRun())
def test_initializeSkiaGoldAttributes_tryjobMissingBuildbucket(self): args = createSkiaGoldArgs(git_revision='a', gerrit_issue=1, gerrit_patchset=2) with self.assertRaises(RuntimeError): skia_gold_properties.SkiaGoldProperties(args)
def test_initializeSkiaGoldAttributes_explicitGitRevision(self): args = createSkiaGoldArgs(git_revision='a') sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {'git_revision': 'a'})
def test_initializeSkiaGoldAttributes_bypassExplicitTrue(self): args = createSkiaGoldArgs(bypass_skia_gold_functionality=True) sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {'bypass_skia_gold_functionality': True})
def test_initializeSkiaGoldAttributes_explicitNoLuciAuth(self): args = createSkiaGoldArgs(no_luci_auth=True) sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {'no_luci_auth': True})
def testGetGitRevision_noExplicitOnBot(self): args = createSkiaGoldArgs(local_pixel_tests=False) sgp = skia_gold_properties.SkiaGoldProperties(args) with self.assertRaises(RuntimeError): _ = sgp.git_revision
def test_initializeSkiaGoldAttributes_explicitNonLocal(self): args = createSkiaGoldArgs(local_pixel_tests=False) sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': False})
def test_initializeSkiaGoldAttributes_explicitCis(self): args = createSkiaGoldArgs(continuous_integration_system='foo') sgp = skia_gold_properties.SkiaGoldProperties(args) self.verifySkiaGoldProperties(sgp, {'continuous_integration_system': 'foo'})