def testNoFailedAttempts(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
self.assertFalse(state.FirstOrLastChangeFailed())
def testDifferentWithMidpoint(self):
quests = [
quest_test.QuestByChange({
change_test.Change(1):
quest_test.QuestPass(),
change_test.Change(9):
quest_test.QuestFail(),
})
]
state = job_state.JobState(quests,
comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# The Changes are different. Add the midpoint.
expected = [
change_test.Change(1),
change_test.Change(5),
change_test.Change(9)
]
self.assertEqual(state._changes, expected)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(5)])
attempt_count_3 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
self.assertEqual(attempt_count_2, attempt_count_3)
self.assertEqual([], state.Differences())
def testDeleteExpiredIsolate(self):
isolate_infos = (
('Mac Builder Perf', change_test.Change(0), 'target_name',
'https://isolate.server', '123'),
('Mac Builder Perf', change_test.Change(1), 'target_name',
'https://isolate.server', '456'),
)
isolate.Put(isolate_infos)
cur = ndb.Key(
'Isolate',
isolate._Key('Mac Builder Perf', change_test.Change(0),
'target_name')).get()
cur.created = datetime.datetime.now() - datetime.timedelta(hours=1)
cur.put()
cur = ndb.Key(
'Isolate',
isolate._Key(isolate_infos[1][0], isolate_infos[1][1],
isolate_infos[1][2])).get()
cur.created = datetime.datetime.now() - (
isolate.ISOLATE_EXPIRY_DURATION + datetime.timedelta(hours=1))
cur.put()
isolate.DeleteExpiredIsolates()
q = isolate.Isolate.query()
isolates = q.fetch()
self.assertEqual(1, len(isolates))
self.assertEqual('123', isolates[0].isolate_hash)
def testFailedAttempt(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
for attempt in state._attempts[change_test.Change(1)]:
attempt._last_execution._exception = "Failed"
self.assertTrue(state.FirstOrLastChangeFailed())
def testDifferences(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(
quests, comparison_mode=job_state.FUNCTIONAL, comparison_magnitude=0.2)
change_a = change_test.Change(1)
change_b = change_test.Change(9)
state.AddChange(change_a)
state.AddChange(change_b)
self.assertEqual([], state.Differences())
def testNoAttempts(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
# It shouldn't happen that a change has no attempts, but we should cope
# gracefully if that somehow happens.
del state._attempts[change_test.Change(1)][:]
del state._attempts[change_test.Change(2)][:]
self.assertFalse(state.FirstOrLastChangeFailed())
def testPutAndGet(self):
isolate_infos = (
('Mac Builder Perf', change_test.Change(1), 'target_name', '7c7e90be'),
('Mac Builder Perf', change_test.Change(2), 'target_name', '38e2f262'),
)
isolate.Put('https://isolate.server', isolate_infos)
isolate_server, isolate_hash = isolate.Get(
'Mac Builder Perf', change_test.Change(1), 'target_name')
self.assertEqual(isolate_server, 'https://isolate.server')
self.assertEqual(isolate_hash, '7c7e90be')
def testPending(self):
quests = [quest_test.QuestSpin()]
state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.Explore()
# The results are pending. Do not add any Attempts or Changes.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
def testExpiredCASReference(self, mock_datetime):
cas_references = (('Mac Builder Perf', change_test.Change(1),
'target_name', 'https://isolate.server',
'7c7e90be'), )
cas.Put(cas_references)
# Teleport to the future after the isolate is expired.
mock_datetime.datetime.utcnow.return_value = (
datetime.datetime.utcnow() + cas.CAS_EXPIRY_DURATION +
datetime.timedelta(days=1))
mock_datetime.timedelta = datetime.timedelta
with self.assertRaises(KeyError):
cas.Get('Mac Builder Perf', change_test.Change(1), 'target_name')
def testPutAndGet(self):
cas_references = (
('Mac Builder Perf', change_test.Change(1), 'target_name',
'https://isolate.server', '7c7e90be'),
('Mac Builder Perf', change_test.Change(2), 'target_name',
'https://isolate.server', '38e2f262'),
)
cas.Put(cas_references)
cas_instance, cas_digest = cas.Get('Mac Builder Perf',
change_test.Change(1),
'target_name')
self.assertEqual(cas_instance, 'https://isolate.server')
self.assertEqual(cas_digest, '7c7e90be')
def testBuilderNameMap(self):
# TODO(dtu): Remove 6 months after LUCI migration is complete.
isolate_infos = (('android_arm64-builder-perf', change_test.Change(1),
'target_name', 'https://isolate.server',
'abcd1234'), )
isolate.Put(isolate_infos)
isolate_server, isolate_hash = isolate.Get(
'Android arm64 Compile Perf', change_test.Change(1), 'target_name')
self.assertEqual(isolate_server, 'https://isolate.server')
self.assertEqual(isolate_hash, 'abcd1234')
with self.assertRaises(KeyError):
isolate.Get('Android arm64 Compile Perf', change_test.Change(2),
'target_name')
def testExpiredIsolate(self, mock_datetime):
isolate_infos = (('Mac Builder Perf', change_test.Change(1),
'target_name', 'https://isolate.server',
'7c7e90be'), )
isolate.Put(isolate_infos)
# Teleport to the future after the isolate is expired.
mock_datetime.datetime.now.return_value = (
datetime.datetime.now() + isolate.ISOLATE_EXPIRY_DURATION +
datetime.timedelta(days=1))
mock_datetime.timedelta = datetime.timedelta
with self.assertRaises(KeyError):
isolate.Get('Mac Builder Perf', change_test.Change(1),
'target_name')
def setUp(self):
super(_FindIsolateExecutionTest, self).setUp()
change = change_test.Change(123)
isolate.Put(
'https://isolate.server',
(('Mac Builder', change, 'telemetry_perf_tests', '7c7e90be'),))
def testSimultaneousBuilds(self, put, get_job_status):
# Two builds started at the same time on the same Change should reuse the
# same build request.
change = change_test.Change(0)
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution_1 = quest.Start(change)
execution_2 = quest.Start(change)
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution_1.Poll()
execution_2.Poll()
self.assertFalse(execution_1.completed)
self.assertFalse(execution_2.completed)
self.assertEqual(put.call_count, 1)
# Check build status.
get_job_status.return_value = {'build': {'status': 'STARTED'}}
execution_1.Poll()
execution_2.Poll()
self.assertFalse(execution_1.completed)
self.assertFalse(execution_2.completed)
self.assertEqual(get_job_status.call_count, 2)
# Look up isolate hash.
isolate.Put(
'https://isolate.server',
(('Mac Builder', change, 'telemetry_perf_tests', 'isolate git hash'),))
execution_1.Poll()
execution_2.Poll()
self.assertExecutionSuccess(execution_1)
self.assertExecutionSuccess(execution_2)
def testBuildNoBucket(self, put, _):
change = change_test.Change(123, 456, patch=True)
quest = find_isolate.FindIsolate(
'Mac Builder', 'telemetry_perf_tests', 'luci.bucket')
execution = quest.Start(change)
del execution._bucket
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
self.assertFalse(execution.completed)
put.assert_called_once_with(
find_isolate.BUCKET,
[
'buildset:patch/gerrit/codereview.com/567890/5',
'buildset:commit/gitiles/chromium.googlesource.com/'
'project/name/+/commit_123'
],
{
'builder_name': 'Mac Builder',
'properties': {
'clobber': True,
'revision': 'commit_123',
'deps_revision_overrides': {test.CATAPULT_URL: 'commit_456'},
'patch_gerrit_url': 'https://codereview.com',
'patch_issue': 567890,
'patch_project': 'project/name',
'patch_ref': 'refs/changes/90/567890/5',
'patch_repository_url': test.CHROMIUM_URL,
'patch_set': 5,
'patch_storage': 'gerrit',
}
}
)
def testPin(self):
request = dict(_BASE_REQUEST)
request['pin'] = 'https://codereview.com/c/foo/bar/+/123'
response = self.Post('/api/new', request, status=200)
job = job_module.JobFromId(json.loads(response.body)['jobId'])
self.assertEqual(job.state._pin, change_test.Change(patch=True))
def testWorkLeft(self):
quests = [
quest_test.QuestCycle(quest_test.QuestPass(), quest_test.QuestSpin())
]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(123))
self.assertTrue(state.ScheduleWork())
def testIsolateLookupSuccess(self):
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution = quest.Start(change_test.Change(123))
execution.Poll()
expected_result_arguments = {
'isolate_server': 'https://isolate.server',
'isolate_hash': '7c7e90be',
}
expected_as_dict = {
'completed': True,
'exception': None,
'details': [
{
'key': 'builder',
'value': 'Mac Builder',
},
{
'key': 'isolate',
'value': '7c7e90be',
'url': 'https://isolate.server/browse?digest=7c7e90be',
},
],
}
self.assertExecutionSuccess(execution)
self.assertEqual(execution.result_values, ())
self.assertEqual(execution.result_arguments, expected_result_arguments)
self.assertEqual(execution.AsDict(), expected_as_dict)
def testIsolateLookupSuccess(self):
quest = find_isolate.FindIsolate(
'Mac Builder', 'telemetry_perf_tests', 'luci.bucket')
# Propagate a thing that looks like a job.
quest.PropagateJob(
FakeJob('cafef00d', 'https://pinpoint/cafef00d', 'performance',
'*****@*****.**'))
execution = quest.Start(change_test.Change(123))
execution.Poll()
expected_result_arguments = {
'isolate_server': 'https://isolate.server',
'isolate_hash': '7c7e90be',
}
expected_as_dict = {
'completed': True,
'exception': None,
'details': [
{
'key': 'builder',
'value': 'Mac Builder',
},
{
'key': 'isolate',
'value': '7c7e90be',
'url': 'https://isolate.server/browse?digest=7c7e90be',
},
],
}
self.assertExecutionSuccess(execution)
self.assertEqual(execution.result_values, ())
self.assertEqual(execution.result_arguments, expected_result_arguments)
self.assertEqual(execution.AsDict(), expected_as_dict)
def testBuildSucceededButIsolateIsMissing(self, put, get_job_status):
quest = find_isolate.FindIsolate(
'Mac Builder', 'telemetry_perf_tests', 'luci.bucket')
execution = quest.Start(change_test.Change(0))
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
# Check build status.
get_job_status.return_value = {
'build': {
'status': 'COMPLETED',
'result': 'SUCCESS',
'result_details_json': """{
"properties": {
"got_revision_cp": "refs/heads/master@{#123}",
"isolate_server": "https://isolate.server",
"swarm_hashes_refs/heads/master(at){#123}_without_patch": {}
}
}""",
}
}
with self.assertRaises(errors.BuildIsolateNotFound):
execution.Poll()
def testAllAttemptsFail(self):
quests = [quest_test.QuestCycle(
quest_test.QuestFail, quest_test.QuestFail, quest_test.QuestFail2)]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(123))
expected_regexp = '7/10.*\nException: Expected error for testing.$'
self.assertTrue(state.ScheduleWork())
with self.assertRaisesRegexp(Exception, expected_regexp):
self.assertFalse(state.ScheduleWork())
def testUnknown(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests,
comparison_mode=job_state.FUNCTIONAL,
comparison_magnitude=0.2)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# Need more information. Add more attempts to one Change.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertGreaterEqual(attempt_count_1, attempt_count_2)
state.ScheduleWork()
state.Explore()
# We still need more information. Add more attempts to the other Change.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
def testSimultaneousBuilds(self, put, get_job_status):
# Two builds started at the same time on the same Change should reuse the
# same build request.
change = change_test.Change(0)
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests',
'luci.bucket')
execution_1 = quest.Start(change)
execution_2 = quest.Start(change)
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution_1.Poll()
execution_2.Poll()
self.assertFalse(execution_1.completed)
self.assertFalse(execution_2.completed)
self.assertEqual(put.call_count, 1)
# Check build status.
get_job_status.return_value = {'build': {'status': 'STARTED'}}
execution_1.Poll()
execution_2.Poll()
self.assertFalse(execution_1.completed)
self.assertFalse(execution_2.completed)
self.assertEqual(get_job_status.call_count, 2)
# Look up isolate hash.
get_job_status.return_value = {
'build': {
'status':
'COMPLETED',
'result':
'SUCCESS',
'result_details_json':
"""{
"properties": {
"got_revision_cp": "refs/heads/master@{#123}",
"isolate_server": "https://isolate.server",
"swarm_hashes_refs/heads/master(at){#123}_without_patch":
{"telemetry_perf_tests": "isolate git hash"}
}
}""",
}
}
execution_1.Poll()
execution_2.Poll()
self.assertExecutionSuccess(execution_1)
self.assertExecutionSuccess(execution_2)
def testBuildNoReviewUrl(self, put, _):
change = change_test.Change(123, 456, patch=True)
results = change.base_commit.AsDict()
del results['review_url']
change.base_commit.AsDict = mock.MagicMock(return_value=results)
quest = find_isolate.FindIsolate(
'Mac Builder', 'telemetry_perf_tests', 'luci.bucket')
execution = quest.Start(change)
del execution._bucket
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
self.assertExecutionFailure(execution, errors.BuildGerritUrlNotFound)
def testBuildSucceededButIsolateIsMissing(self, put, get_job_status):
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution = quest.Start(change_test.Change(0))
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
# Check build status.
get_job_status.return_value = {
'build': {
'status': 'COMPLETED',
'result': 'SUCCESS',
}
}
with self.assertRaises(find_isolate.IsolateNotFoundError):
execution.Poll()
def testBuildCanceled(self, put, get_job_status):
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution = quest.Start(change_test.Change(0))
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
# Check build status.
get_job_status.return_value = {
'build': {
'status': 'COMPLETED',
'result': 'CANCELED',
'cancelation_reason': 'TIMEOUT',
}
}
execution.Poll()
self.assertExecutionFailure(execution, find_isolate.BuildError)
def testDifferentNoMidpoint(self):
quests = [
quest_test.QuestByChange({
change_test.Change(1): quest_test.QuestPass(),
change_test.Change(2): quest_test.QuestFail(),
})
]
state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
state.Explore()
# The Changes are different, but there's no midpoint. We're done.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(2)])
self.assertEqual(attempt_count_1, attempt_count_2)
self.assertEqual([(change_test.Change(1), change_test.Change(2))],
state.Differences())
def testBuildFailure(self, put, get_job_status):
quest = find_isolate.FindIsolate(
'Mac Builder', 'telemetry_perf_tests', 'luci.bucket')
execution = quest.Start(change_test.Change(0))
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
# Check build status.
get_job_status.return_value = {
'build': {
'status': 'COMPLETED',
'result': 'FAILURE',
'failure_reason': 'BUILD_FAILURE',
}
}
execution.Poll()
self.assertExecutionFailure(execution, errors.BuildFailed)
def testSame(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests, comparison_mode=job_state.FUNCTIONAL)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
for _ in xrange(5):
# More Attempts give more confidence that they are, indeed, the same.
state.AddAttempts(change_test.Change(1))
state.AddAttempts(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# The Changes are the same. Do not add any Attempts or Changes.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
def testBuildLifecycle(self, put, get_job_status):
change = change_test.Change(123, 456, patch=True)
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution = quest.Start(change)
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
self.assertFalse(execution.completed)
put.assert_called_once_with(find_isolate.BUCKET, {
'builder_name': 'Mac Builder',
'properties': {
'clobber': True,
'parent_got_revision': 'commit_123',
'deps_revision_overrides': {test.CATAPULT_URL: 'commit_456'},
'patch_gerrit_url': 'https://codereview.com',
'patch_issue': 567890,
'patch_project': 'project/name',
'patch_ref': 'refs/changes/90/567890/5',
'patch_repository_url': test.CHROMIUM_URL,
'patch_set': 5,
'patch_storage': 'gerrit',
}
})
# Check build status.
get_job_status.return_value = {'build': {
'status': 'STARTED',
'url': 'build_url',
}}
execution.Poll()
self.assertFalse(execution.completed)
get_job_status.assert_called_once_with('build_id')
# Look up isolate hash.
isolate.Put((
('Mac Builder', change, 'telemetry_perf_tests',
'https://isolate.server', 'isolate git hash'),
))
execution.Poll()
expected_result_arguments = {
'isolate_server': 'https://isolate.server',
'isolate_hash': 'isolate git hash',
}
expected_as_dict = {
'completed': True,
'exception': None,
'details': [
{
'key': 'builder',
'value': 'Mac Builder',
},
{
'key': 'build',
'value': 'build_id',
'url': 'build_url',
},
{
'key': 'isolate',
'value': 'isolate git hash',
'url': 'https://isolate.server/browse?digest=isolate git hash',
},
],
}
self.assertExecutionSuccess(execution)
self.assertEqual(execution.result_values, ())
self.assertEqual(execution.result_arguments, expected_result_arguments)
self.assertEqual(execution.AsDict(), expected_as_dict)
