def test_noLinkOnSuccess(self, cmd_mock):
cmd_mock.return_value = (0, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 0)
comparison_result = session._comparison_results['name']
self.assertEqual(comparison_result.public_triage_link, None)
self.assertEqual(comparison_result.internal_triage_link, None)
self.assertNotEqual(comparison_result.triage_link_omission_reason,
None)
def test_initFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (1, 'Init failed')
session = skia_gold_session.SkiaGoldSession(self._working_dir, None,
None, None, None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(
status, skia_gold_session.SkiaGoldSession.StatusCodes.INIT_FAILURE)
self.assertEqual(error, 'Init failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 0)
self.assertEqual(diff_mock.call_count, 0)
def test_commandTryjobArgsMissing(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
None, None, None)
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertNotIn('--issue', call_args)
self.assertNotIn('--patchset', call_args)
self.assertNotIn('--jobid', call_args)
self.assertNotIn('--crs', call_args)
self.assertNotIn('--cis', call_args)
def test_authFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (1, 'Auth failed')
sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs())
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
self._json_keys, None,
None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(
status, skia_gold_session.SkiaGoldSession.StatusCodes.AUTH_FAILURE)
self.assertEqual(error, 'Auth failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 0)
self.assertEqual(compare_mock.call_count, 0)
self.assertEqual(diff_mock.call_count, 0)
def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (0, None)
session = skia_gold_session.SkiaGoldSession(self._working_dir, None,
self._json_keys, None,
None)
status, _ = session.RunComparison(None, None, None)
self.assertEqual(status,
skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS)
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_commandTryjobArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
None, None, None)
session.Initialize()
call_args = cmd_mock.call_args[0][0]
assertArgWith(self, call_args, '--issue', '1')
assertArgWith(self, call_args, '--patchset', '2')
assertArgWith(self, call_args, '--jobid', '3')
assertArgWith(self, call_args, '--crs', 'gerrit')
assertArgWith(self, call_args, '--cis', 'buildbucket')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
self._json_keys,
'corpus',
instance='instance')
session.Compare('name', 'png_file')
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('add', call_args)
assertArgWith(self, call_args, '--test-name', 'name')
assertArgWith(self, call_args, '--png-file', 'png_file')
assertArgWith(self, call_args, '--work-dir', self._working_dir)
def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (0, None)
keys_file = os.path.join(self._working_dir, 'keys.json')
with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = skia_gold_session.SkiaGoldSession(self._working_dir, None,
keys_file, None, None)
status, _ = session.RunComparison(None, None, None)
self.assertEqual(status,
skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS)
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
args = createSkiaGoldArgs(local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
self._json_keys, None,
None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(
status, skia_gold_session.SkiaGoldSession.StatusCodes.
COMPARISON_FAILURE_REMOTE)
self.assertEqual(error, 'Compare failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_noOutputManagerLocal(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (0, None)
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
self._json_keys, None,
None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(
status,
skia_gold_session.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER)
self.assertEqual(error, 'No output manager for local diff images')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
def test_validOmissionOnMissingLink(self, cmd_mock):
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
def WriteTriageLinkFile(_):
with open(session._triage_link_file, 'w'):
pass
return (1, None)
cmd_mock.side_effect = WriteTriageLinkFile
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
comparison_result = session._comparison_results['name']
self.assertEqual(comparison_result.public_triage_link, None)
self.assertEqual(comparison_result.internal_triage_link, None)
self.assertIn('Gold did not provide a triage link',
comparison_result.triage_link_omission_reason)
def test_validOmissionOnIoError(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
def DeleteTriageLinkFile(_):
os.remove(session._triage_link_file)
return (1, None)
cmd_mock.side_effect = DeleteTriageLinkFile
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
comparison_result = session._comparison_results['name']
self.assertEqual(comparison_result.public_triage_link, None)
self.assertEqual(comparison_result.internal_triage_link, None)
self.assertNotEqual(comparison_result.triage_link_omission_reason, None)
self.assertIn('Failed to read',
comparison_result.triage_link_omission_reason)
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
'keys_file',
'corpus',
instance='instance')
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('init', call_args)
self.assertIn('--passfail', call_args)
assertArgWith(self, call_args, '--instance', 'instance')
assertArgWith(self, call_args, '--corpus', 'corpus')
assertArgWith(self, call_args, '--keys-file', 'keys_file')
assertArgWith(self, call_args, '--work-dir', self._working_dir)
assertArgWith(self, call_args, '--failure-file', session._triage_link_file)
assertArgWith(self, call_args, '--commit', 'a')
def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (1, 'Diff failed')
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
self._json_keys, None,
None)
status, error = session.RunComparison(None, None,
'Definitely an output manager')
self.assertEqual(
status,
skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE)
self.assertEqual(error, 'Diff failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 1)
def test_clLinkOnTrybot(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
'keys_file',
None,
instance='instance')
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
self.assertNotEqual(session._comparison_results['name'].triage_link,
None)
self.assertEqual(session._comparison_results['name'].triage_link,
'https://instance-gold.skia.org/cl/gerrit/1')
self.assertEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
def test_individualLinkOnCi(self, cmd_mock):
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
def WriteTriageLinkFile(_):
with open(session._triage_link_file, 'w') as f:
f.write('foobar')
return (1, None)
cmd_mock.side_effect = WriteTriageLinkFile
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
self.assertNotEqual(session._comparison_results['name'].triage_link,
None)
self.assertEqual(session._comparison_results['name'].triage_link,
'foobar')
self.assertEqual(
session._comparison_results['name'].triage_link_omission_reason,
None)
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
self._json_keys,
'corpus',
instance='instance')
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('init', call_args)
self.assertIn('--passfail', call_args)
assertArgWith(self, call_args, '--instance', 'instance')
assertArgWith(self, call_args, '--corpus', 'corpus')
# The keys file should have been copied to the working directory.
assertArgWith(self, call_args, '--keys-file',
os.path.join(self._working_dir, 'gold_keys.json'))
assertArgWith(self, call_args, '--work-dir', self._working_dir)
assertArgWith(self, call_args, '--failure-file',
session._triage_link_file)
assertArgWith(self, call_args, '--commit', 'a')
def test_compareForceDryrun(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (0, None)
diff_mock.return_value = (0, None)
args = createSkiaGoldArgs(local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
self._json_keys, None,
None)
status, _ = session.RunComparison(None, None, None, force_dryrun=True)
self.assertEqual(status,
skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS)
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
compare_mock.assert_called_with(name=None,
png_file=mock.ANY,
inexact_matching_args=None,
optional_keys=None,
force_dryrun=True)
def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (0, None)
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
keys_file = os.path.join(self._working_dir, 'keys.json')
with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
keys_file, None, None)
status, error = session.RunComparison(None, None,
'Definitely an output manager')
self.assertEqual(
status, skia_gold_session.SkiaGoldSession.StatusCodes.
COMPARISON_FAILURE_LOCAL)
self.assertEqual(error, 'Compare failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 1)
def test_clLinkOnTrybot(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
self._json_keys,
None,
instance='instance')
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
comparison_result = session._comparison_results['name']
self.assertNotEqual(comparison_result.public_triage_link, None)
self.assertNotEqual(comparison_result.internal_triage_link, None)
internal_link = 'https://instance-gold.skia.org/cl/gerrit/1'
public_link = 'https://instance-public-gold.skia.org/cl/gerrit/1'
self.assertEqual(comparison_result.internal_triage_link, internal_link)
self.assertEqual(comparison_result.public_triage_link, public_link)
self.assertEqual(comparison_result.triage_link_omission_reason, None)
self.assertEqual(session.GetTriageLinks('name'),
(public_link, internal_link))
