def testEvaluateSuccess_NeedToRefineAttempts(self):
self.PopulateSimpleBisectionGraph(self.job)
task_module.Evaluate(
self.job,
event_module.Event(type='initiate', target_task=None, payload={}),
self.BisectionEvaluatorForTesting(
bisection_test_util.FakeReadValueMapResult(
self.job, {
change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': commit
}]
}): values
for commit, values in (
('commit_0', range(10)),
('commit_1', range(1, 11)),
('commit_2', range(2, 12)),
('commit_3', range(3, 13)),
('commit_4', range(3, 13)),
('commit_5', range(3, 13)),
)
})))
# Here we test that we have more than the minimum attempts for the change
# between commit_1 and commit_2.
evaluate_result = task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.Selector(task_type='read_value'))
attempt_counts = {}
for payload in evaluate_result.values():
change = change_module.Change.FromDict(payload.get('change'))
attempt_counts[change] = attempt_counts.get(change, 0) + 1
self.assertGreater(
attempt_counts[change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': 'commit_2',
}]
})], 10)
self.assertLess(
attempt_counts[change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': 'commit_2',
}]
})], 100)
# We know that we will refine the graph until we see the progression from
# commit_0 -> commit_1 -> commit_2 -> commit_3 and stabilize.
evaluate_result = task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.Selector(task_type='find_culprit'))
self.assertIn('performance_bisection', evaluate_result)
self.assertEquals(evaluate_result['performance_bisection']['culprits'],
[mock.ANY, mock.ANY, mock.ANY])
def testEvaluateSuccess_NeedToRefineAttempts(self):
self.PopulateSimpleBisectionGraph(self.job)
task_module.Evaluate(
self.job,
event_module.Event(type='initiate', target_task=None, payload={}),
self.BisectionEvaluatorForTesting(
bisection_test_util.FakeReadValueMapResult(
self.job, {
change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': commit
}]
}): values for commit, values in (
('commit_0', range(10)),
('commit_1', range(10)),
('commit_2', range(4, 14)),
('commit_3', range(3, 13)),
('commit_4', range(3, 13)),
('commit_5', range(3, 13)),
)
})))
# Here we test that we have more than the minimum attempts for the change
# between commit_1 and commit_2.
evaluate_result = task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.Selector(task_type='read_value'))
attempt_counts = {}
for payload in evaluate_result.values():
change = change_module.Change.FromDict(payload.get('change'))
attempt_counts[change] = attempt_counts.get(change, 0) + 1
self.assertGreater(
attempt_counts[change_module.Change.FromDict(
{'commits': [{
'repository': 'chromium',
'git_hash': 'commit_2',
}]})], 10)
self.assertLess(
attempt_counts[change_module.Change.FromDict(
{'commits': [{
'repository': 'chromium',
'git_hash': 'commit_2',
}]})], 100)
# We know that we will never get a deterministic answer, so we ensure that
# we don't inadvertently blame the wrong changes at the end of the
# refinement.
evaluate_result = task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.Selector(task_type='find_culprit'))
self.assertIn('performance_bisection', evaluate_result)
logging.info('Results: %s', evaluate_result['performance_bisection'])
self.assertEquals(evaluate_result['performance_bisection']['culprits'], [])
def testEvaluateFailure_DependenciesNoResults(self):
self.PopulateSimpleBisectionGraph(self.job)
task_module.Evaluate(
self.job,
event_module.Event(type='initiate', target_task=None, payload={}),
self.BisectionEvaluatorForTesting(
bisection_test_util.FakeReadValueSameResult(self.job, None)))
evaluate_result = task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.Selector(task_type='find_culprit'))
self.assertIn('performance_bisection', evaluate_result)
self.assertEqual(evaluate_result['performance_bisection']['status'],
'failed')
self.assertNotEqual([], evaluate_result['performance_bisection']['errors'])
def testEvaluateSuccess_SpeculateBisection(self):
self.PopulateSimpleBisectionGraph(self.job)
task_module.Evaluate(
self.job,
event_module.Event(type='initiate', target_task=None, payload={}),
self.BisectionEvaluatorForTesting(
bisection_test_util.FakeReadValueMapResult(
self.job, {
change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': commit
}]
}): values
for commit, values in (
('commit_0', [1.0] * 10),
('commit_1', [1.0] * 10),
('commit_2', [2.0] * 10),
('commit_3', [2.0] * 10),
('commit_4', [2.0] * 10),
('commit_5', [2.0] * 10),
)
})))
evaluate_result = task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.Selector(task_type='find_culprit'))
self.assertIn('performance_bisection', evaluate_result)
logging.info('Results: %s', evaluate_result['performance_bisection'])
# Here we're testing that we can find the change between commit_1 and
# commit_2 in the values we seed above.
self.assertEquals(evaluate_result['performance_bisection']['culprits'],
[[
change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': 'commit_1'
}]
}).AsDict(),
change_module.Change.FromDict({
'commits': [{
'repository': 'chromium',
'git_hash': 'commit_2'
}]
}).AsDict()
]])
def testSerializeJob(self):
self.PopulateSimpleBisectionGraph(self.job)
task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.SequenceEvaluator([
evaluators.DispatchByTaskType({
'find_isolate':
bisection_test_util.FakeFoundIsolate(self.job),
'run_test':
bisection_test_util.FakeSuccessfulRunTest(self.job),
'read_value':
bisection_test_util.FakeReadValueSameResult(self.job, 1.0),
'find_culprit':
performance_bisection.Evaluator(self.job),
}),
evaluators.TaskPayloadLiftingEvaluator()
]))
logging.debug('Finished evaluating job state.')
job_dict = self.job.AsDict(options=[job_module.OPTION_STATE])
logging.debug('Job = %s', pprint.pformat(job_dict))
self.assertTrue(self.job.use_execution_engine)
self.assertEqual(
{
'arguments':
mock.ANY,
'bug_id':
None,
'cancel_reason':
None,
'comparison_mode':
'performance',
'configuration':
'some_configuration',
'created':
mock.ANY,
'difference_count':
0,
'exception':
None,
'job_id':
mock.ANY,
'metric':
'some_benchmark',
'name':
mock.ANY,
'quests': ['Build', 'Test', 'Get results'],
'results_url':
mock.ANY,
'status':
mock.ANY,
'updated':
mock.ANY,
'user':
None,
# NOTE: Here we're asseessing the structure of the results, not the
# actual contents. We'll reserve more specific content form testing
# in other test cases, but for now we're ensuring that we're able to
# get the shape of the data in a certain way.
'state': [{
'attempts': [{
'executions': [mock.ANY] * 3
}] + [{
'executions': [None, mock.ANY, mock.ANY]
}] * 9,
'change':
self.start_change.AsDict(),
'comparisons': {
'prev': None,
'next': 'same',
},
'result_values': [mock.ANY] * 10,
}, {
'attempts': [{
'executions': [mock.ANY] * 3
}] + [{
'executions': [None, mock.ANY, mock.ANY]
}] * 9,
'change':
self.end_change.AsDict(),
'comparisons': {
'prev': 'same',
'next': None,
},
'result_values': [mock.ANY] * 10,
}]
},
job_dict)
