def testNoFailedAttempts(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
self.assertFalse(state.FirstOrLastChangeFailed())
def testDifferentWithMidpoint(self):
quests = [
quest_test.QuestByChange({
change_test.Change(1):
quest_test.QuestPass(),
change_test.Change(9):
quest_test.QuestFail(),
})
]
state = job_state.JobState(quests,
comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# The Changes are different. Add the midpoint.
expected = [
change_test.Change(1),
change_test.Change(5),
change_test.Change(9)
]
self.assertEqual(state._changes, expected)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(5)])
attempt_count_3 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
self.assertEqual(attempt_count_2, attempt_count_3)
self.assertEqual([], state.Differences())
def testUnknown(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests,
comparison_mode=job_state.FUNCTIONAL,
comparison_magnitude=0.2)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# Need more information. Add more attempts to one Change.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertGreaterEqual(attempt_count_1, attempt_count_2)
state.ScheduleWork()
state.Explore()
# We still need more information. Add more attempts to the other Change.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
def New(cls, arguments, quests, auto_explore, bug_id=None, tags=None):
# Create job.
return cls(arguments=arguments,
auto_explore=auto_explore,
bug_id=bug_id,
tags=tags,
state=job_state.JobState(quests))
def testWorkLeft(self):
quests = [
quest_test.QuestCycle(quest_test.QuestPass(), quest_test.QuestSpin())
]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(123))
self.assertTrue(state.ScheduleWork())
def New(cls, quests, changes, arguments=None, auto_explore=False,
bug_id=None, comparison_mode=None, pin=None, tags=None, user=None):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
auto_explore: If True, the Job should automatically add additional
Attempts and Changes based on comparison of results values.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: A member of the ComparisonMode enum, which the Job uses
to figure out whether to perform a functional or performance bisect.
If None, the Job will not automatically add any Attempts or Changes.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
Returns:
A Job object.
"""
job = cls(state=job_state.JobState(quests, pin=pin),
arguments=arguments or {},
auto_explore=auto_explore,
bug_id=bug_id,
comparison_mode=comparison_mode,
tags=tags,
user=user)
for c in changes:
job.AddChange(c)
job.put()
return job
def testAllAttemptsFail(self):
q = _QuestStub(_ExecutionFail, _ExecutionFail, _ExecutionFail2)
state = job_state.JobState([q])
state.AddChange('change')
expected_regexp = '7/10.*\nException: Expected error for testing.$'
self.assertTrue(state.ScheduleWork())
with self.assertRaisesRegexp(Exception, expected_regexp):
self.assertFalse(state.ScheduleWork())
def New(cls,
quests,
changes,
arguments=None,
bug_id=None,
comparison_mode=None,
comparison_magnitude=None,
gerrit_server=None,
gerrit_change_id=None,
name=None,
pin=None,
tags=None,
user=None):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: Either 'functional' or 'performance', which the Job uses
to figure out whether to perform a functional or performance bisect.
If None, the Job will not automatically add any Attempts or Changes.
comparison_magnitude: The estimated size of the regression or improvement
to look for. Smaller magnitudes require more repeats.
gerrit_server: Server of the Gerrit code review to update with job
results.
gerrit_change_id: Change id of the Gerrit code review to update with job
results.
name: The user-provided name of the Job.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
Returns:
A Job object.
"""
state = job_state.JobState(quests,
comparison_mode=comparison_mode,
comparison_magnitude=comparison_magnitude,
pin=pin)
job = cls(state=state,
arguments=arguments or {},
bug_id=bug_id,
comparison_mode=comparison_mode,
gerrit_server=gerrit_server,
gerrit_change_id=gerrit_change_id,
name=name,
tags=tags,
user=user,
started=False,
cancelled=False)
for c in changes:
job.AddChange(c)
job.put()
return job
def testFailedAttempt(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
for attempt in state._attempts[change_test.Change(1)]:
attempt._last_execution._exception = "Failed"
self.assertTrue(state.FirstOrLastChangeFailed())
def testAllAttemptsFail(self):
quests = [quest_test.QuestCycle(
quest_test.QuestFail, quest_test.QuestFail, quest_test.QuestFail2)]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(123))
expected_regexp = '7/10.*\nException: Expected error for testing.$'
self.assertTrue(state.ScheduleWork())
with self.assertRaisesRegexp(Exception, expected_regexp):
self.assertFalse(state.ScheduleWork())
def testDifferences(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(
quests, comparison_mode=job_state.FUNCTIONAL, comparison_magnitude=0.2)
change_a = change_test.Change(1)
change_b = change_test.Change(9)
state.AddChange(change_a)
state.AddChange(change_b)
self.assertEqual([], state.Differences())
def testNoAttempts(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
# It shouldn't happen that a change has no attempts, but we should cope
# gracefully if that somehow happens.
del state._attempts[change_test.Change(1)][:]
del state._attempts[change_test.Change(2)][:]
self.assertFalse(state.FirstOrLastChangeFailed())
def testPending(self):
quests = [quest_test.QuestSpin()]
state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.Explore()
# The results are pending. Do not add any Attempts or Changes.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
def New(cls,
arguments,
quests,
auto_explore,
comparison_mode=None,
user=None,
bug_id=None,
tags=None):
# Create job.
return cls(arguments=arguments,
auto_explore=auto_explore,
comparison_mode=comparison_mode,
user=user,
bug_id=bug_id,
tags=tags,
state=job_state.JobState(quests))
def testDifferentNoMidpoint(self):
quests = [quest_test.QuestByChange({
change_test.Change(1): quest_test.QuestPass(),
change_test.Change(2): quest_test.QuestFail(),
})]
state = job_state.JobState(quests, comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(2))
state.ScheduleWork()
state.Explore()
# The Changes are different, but there's no midpoint. We're done.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(2)])
self.assertEqual(attempt_count_1, attempt_count_2)
def testSame(self):
quests = [quest_test.QuestPass()]
state = job_state.JobState(quests, comparison_mode=job_state.FUNCTIONAL)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
for _ in xrange(5):
# More Attempts give more confidence that they are, indeed, the same.
state.AddAttempts(change_test.Change(1))
state.AddAttempts(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# The Changes are the same. Do not add any Attempts or Changes.
self.assertEqual(len(state._changes), 2)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
def testDifferentWithMidpoint(self):
quests = [
quest_test.QuestByChange({
change_test.Change(1):
quest_test.QuestPass(),
change_test.Change(9):
quest_test.QuestFail(),
})
]
state = job_state.JobState(quests,
comparison_mode=job_state.PERFORMANCE)
state.AddChange(change_test.Change(1))
state.AddChange(change_test.Change(9))
state.ScheduleWork()
state.Explore()
# The Changes are different. Add the speculated modpoints in a two-level
# deep bisection tree, which include 3, 5 (the midpoint between 0 and 9),
# and 7.
expected = [
change_test.Change(1),
change_test.Change(3),
change_test.Change(5),
change_test.Change(7),
change_test.Change(9),
]
# View the whole diff in case of failure.
self.maxDiff = None
self.assertEqual(state._changes, expected)
attempt_count_1 = len(state._attempts[change_test.Change(1)])
attempt_count_2 = len(state._attempts[change_test.Change(3)])
attempt_count_3 = len(state._attempts[change_test.Change(5)])
attempt_count_4 = len(state._attempts[change_test.Change(7)])
attempt_count_5 = len(state._attempts[change_test.Change(9)])
self.assertEqual(attempt_count_1, attempt_count_2)
self.assertEqual(attempt_count_2, attempt_count_3)
self.assertEqual(attempt_count_3, attempt_count_4)
self.assertEqual(attempt_count_4, attempt_count_5)
self.assertEqual([], state.Differences())
def New(cls,
quests,
changes,
arguments=None,
bug_id=None,
comparison_mode=None,
comparison_magnitude=None,
gerrit_server=None,
gerrit_change_id=None,
name=None,
pin=None,
tags=None,
user=None,
use_execution_engine=False):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: Either 'functional' or 'performance', which the Job uses
to figure out whether to perform a functional or performance bisect. If
None, the Job will not automatically add any Attempts or Changes.
comparison_magnitude: The estimated size of the regression or improvement
to look for. Smaller magnitudes require more repeats.
gerrit_server: Server of the Gerrit code review to update with job
results.
gerrit_change_id: Change id of the Gerrit code review to update with job
results.
name: The user-provided name of the Job.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
use_execution_engine: A bool indicating whether to use the experimental
execution engine. Currently defaulted to False, but will be switched to
True and eventually removed as an option later.
Returns:
A Job object.
"""
state = job_state.JobState(quests,
comparison_mode=comparison_mode,
comparison_magnitude=comparison_magnitude,
pin=pin)
args = arguments or {}
job = cls(state=state,
arguments=args,
bug_id=bug_id,
comparison_mode=comparison_mode,
gerrit_server=gerrit_server,
gerrit_change_id=gerrit_change_id,
name=name,
tags=tags,
user=user,
started=False,
cancelled=False,
use_execution_engine=use_execution_engine)
# Pull out the benchmark arguments to the top-level.
job.benchmark_arguments = BenchmarkArguments.FromArgs(args)
if use_execution_engine:
# Short-circuit the process because we don't need any further processing
# here when we're using the execution engine.
job.put()
return job
for c in changes:
job.AddChange(c)
job.put()
# At this point we already have an ID, so we should go through each of the
# quests associated with the state, and provide the Job ID through a common
# API.
job.state.PropagateJob(job)
job.put()
return job
def testNoWorkLeft(self): quests = [quest_test.QuestPass()] state = job_state.JobState(quests) state.AddChange(change_test.Change(123)) self.assertTrue(state.ScheduleWork()) self.assertFalse(state.ScheduleWork())
def testNoAttempts(self): state = job_state.JobState(()) self.assertFalse(state.ScheduleWork())
def testNoChanges(self):
state = job_state.JobState(())
self.assertFalse(state.FirstOrLastChangeFailed())
def testNoWorkLeft(self):
state = job_state.JobState([_QuestStub(_ExecutionPass)])
state.AddChange('change')
self.assertTrue(state.ScheduleWork())
self.assertFalse(state.ScheduleWork())
