def testEvaluateFailedDependency(self, *_):
self.PopulateTaskGraph(
benchmark='some_benchmark',
chart='chart',
trace='must_not_be_found',
mode='graph_json')
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate', target_task=None, payload={}),
evaluators.SequenceEvaluator(
evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(
bisection_test_util.FakeFoundIsolate(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('run_test'),
delegate=evaluators.SequenceEvaluator(
evaluators=(
bisection_test_util.FakeFailedRunTest(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
read_value.Evaluator(self.job),
))))
self.assertEqual(
{
'read_value_chromium@aaaaaaa_%s' % (attempt,): {
'benchmark': 'some_benchmark',
'change': mock.ANY,
'mode': 'graph_json',
'results_filename': 'some_benchmark/perf_results.json',
'results_path': ['some_benchmark', 'perf_results.json'],
'histogram_options': {
'grouping_label': None,
'story': None,
'statistic': None,
'histogram_name': 'chart',
},
'graph_json_options': {
'chart': 'chart',
'trace': 'must_not_be_found',
},
'errors': [{
'reason': 'DependencyFailed',
'message': mock.ANY,
}],
'status': 'failed',
'tries': 1,
'index': attempt,
} for attempt in range(10)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select', target_task=None, payload={}),
evaluators.Selector(task_type='read_value')))
def setUp(self):
super(EvaluatorTest, self).setUp()
self.maxDiff = None
self.job = job_module.Job.New((), ())
# Set up a common evaluator for all the test cases.
self.evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(bisection_test_util.FakeFoundIsolate(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('run_test'),
delegate=evaluators.SequenceEvaluator(evaluators=(
bisection_test_util.FakeSuccessfulRunTest(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
read_value.Evaluator(self.job),
))
def testEvaluateToCompletion(self, swarming_task_result,
swarming_tasks_new):
swarming_tasks_new.return_value = {'task_id': 'task id'}
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(bisection_test_util.FakeFoundIsolate(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
self.assertNotEqual({},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
# Ensure that we've found all the 'run_test' tasks.
self.assertEqual(
{
'run_test_chromium@aaaaaaa_%s' % (attempt, ): {
'status': 'ongoing',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
'service_account': mock.ANY,
},
'swarming_task_id': 'task id',
'tries': 1,
'change': mock.ANY,
'index': attempt,
}
for attempt in range(10)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select', target_task=None,
payload={}),
evaluators.Selector(task_type='run_test')))
# Ensure that we've actually made the calls to the Swarming service.
swarming_tasks_new.assert_called()
self.assertGreaterEqual(swarming_tasks_new.call_count, 10)
# Then we propagate an event for each of the run_test tasks in the graph.
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 0,
'failure': False,
'outputs_ref': {
'isolatedserver': 'output isolate server',
'isolated': 'output isolate hash',
},
'state': 'COMPLETED',
}
for attempt in range(10):
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(
type='update',
target_task='run_test_chromium@aaaaaaa_%s' %
(attempt, ),
payload={}), evaluator), 'Attempt #%s' % (attempt, ))
# Ensure that we've polled the status of each of the tasks, and that we've
# marked the tasks completed.
self.assertEqual(
{
'run_test_chromium@aaaaaaa_%s' % (attempt, ): {
'status': 'completed',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
'service_account': mock.ANY,
},
'swarming_task_result': {
'bot_id': mock.ANY,
'state': 'COMPLETED',
'failure': False,
},
'isolate_server': 'output isolate server',
'isolate_hash': 'output isolate hash',
'swarming_task_id': 'task id',
'tries': 1,
'change': mock.ANY,
'index': attempt,
}
for attempt in range(10)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select', target_task=None,
payload={}),
evaluators.Selector(task_type='run_test')))
# Ensure that we've actually made the calls to the Swarming service.
swarming_task_result.assert_called()
self.assertGreaterEqual(swarming_task_result.call_count, 10)
def testEvaluateHandleFailures_Expired(self, swarming_task_result,
swarming_tasks_new):
swarming_tasks_new.return_value = {'task_id': 'task id'}
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(bisection_test_util.FakeFoundIsolate(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
self.assertNotEqual({},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
swarming_task_result.return_value = {
'state': 'EXPIRED',
}
for attempt in range(10):
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(
type='update',
target_task='run_test_chromium@aaaaaaa_%s' %
(attempt, ),
payload={
'kind': 'pubsub_message',
'action': 'poll'
}), evaluator), 'Attempt #%s' % (attempt, ))
self.assertEqual(
{
'run_test_chromium@aaaaaaa_%s' % (attempt, ): {
'status': 'failed',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'errors': [
{
'reason': 'SwarmingExpired',
'message': mock.ANY
},
],
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
'service_account': mock.ANY,
},
'swarming_task_result': {
'state': 'EXPIRED',
},
'swarming_task_id': 'task id',
'tries': 1,
'change': mock.ANY,
'index': attempt,
}
for attempt in range(10)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select', target_task=None,
payload={}),
evaluators.Selector(task_type='run_test')))
def testEvaluateHandleFailures_Hard(self, swarming_task_stdout,
swarming_task_result,
swarming_tasks_new):
swarming_tasks_new.return_value = {'task_id': 'task id'}
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(bisection_test_util.FakeFoundIsolate(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
self.assertNotEqual({},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
# We set it up so that when we poll the swarming task, that we're going to
# get an error status. We're expecting that hard failures are detected.
swarming_task_stdout.return_value = {
'output':
"""Traceback (most recent call last):
File "../../testing/scripts/run_performance_tests.py", line 282, in <module>
sys.exit(main())
File "../../testing/scripts/run_performance_tests.py", line 226, in main
benchmarks = args.benchmark_names.split(',')
AttributeError: 'Namespace' object has no attribute 'benchmark_names'"""
}
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 1,
'failure': True,
'outputs_ref': {
'isolatedserver': 'output isolate server',
'isolated': 'output isolate hash',
},
'state': 'COMPLETED',
}
for attempt in range(10):
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(
type='update',
target_task='run_test_chromium@aaaaaaa_%s' %
(attempt, ),
payload={
'kind': 'pubsub_message',
'action': 'poll'
}), evaluator), 'Attempt #%s' % (attempt, ))
self.assertEqual(
{
'run_test_chromium@aaaaaaa_%s' % (attempt, ): {
'status': 'failed',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'errors': mock.ANY,
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
'service_account': mock.ANY,
},
'swarming_task_result': {
'bot_id': mock.ANY,
'state': 'COMPLETED',
'failure': True,
},
'isolate_server': 'output isolate server',
'isolate_hash': 'output isolate hash',
'swarming_task_id': 'task id',
'tries': 1,
'change': mock.ANY,
'index': attempt,
}
for attempt in range(10)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select', target_task=None,
payload={}),
evaluators.Selector(task_type='run_test')))
def testSerializeJob(self):
self.PopulateSimpleBisectionGraph(self.job)
task_module.Evaluate(
self.job, bisection_test_util.SelectEvent(),
evaluators.SequenceEvaluator([
evaluators.DispatchByTaskType({
'find_isolate':
bisection_test_util.FakeFoundIsolate(self.job),
'run_test':
bisection_test_util.FakeSuccessfulRunTest(self.job),
'read_value':
bisection_test_util.FakeReadValueSameResult(self.job, 1.0),
'find_culprit':
performance_bisection.Evaluator(self.job),
}),
evaluators.TaskPayloadLiftingEvaluator()
]))
logging.debug('Finished evaluating job state.')
job_dict = self.job.AsDict(options=[job_module.OPTION_STATE])
logging.debug('Job = %s', pprint.pformat(job_dict))
self.assertTrue(self.job.use_execution_engine)
self.assertEqual(
{
'arguments':
mock.ANY,
'bug_id':
None,
'cancel_reason':
None,
'comparison_mode':
'performance',
'configuration':
'some_configuration',
'created':
mock.ANY,
'difference_count':
0,
'exception':
None,
'job_id':
mock.ANY,
'metric':
'some_benchmark',
'name':
mock.ANY,
'quests': ['Build', 'Test', 'Get results'],
'results_url':
mock.ANY,
'status':
mock.ANY,
'updated':
mock.ANY,
'user':
None,
# NOTE: Here we're asseessing the structure of the results, not the
# actual contents. We'll reserve more specific content form testing
# in other test cases, but for now we're ensuring that we're able to
# get the shape of the data in a certain way.
'state': [{
'attempts': [{
'executions': [mock.ANY] * 3
}] + [{
'executions': [None, mock.ANY, mock.ANY]
}] * 9,
'change':
self.start_change.AsDict(),
'comparisons': {
'prev': None,
'next': 'same',
},
'result_values': [mock.ANY] * 10,
}, {
'attempts': [{
'executions': [mock.ANY] * 3
}] + [{
'executions': [None, mock.ANY, mock.ANY]
}] * 9,
'change':
self.end_change.AsDict(),
'comparisons': {
'prev': 'same',
'next': None,
},
'result_values': [mock.ANY] * 10,
}]
},
job_dict)