def testEvaluateFailedDependency(self, *_):
self.PopulateTaskGraph(benchmark='some_benchmark',
chart='chart',
trace='must_not_be_found',
mode='graph_json')
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}),
evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(evaluators=(
functools.partial(FakeFoundIsolate, self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('run_test'),
delegate=evaluators.SequenceEvaluator(evaluators=(
functools.partial(FakeFailedRunTest, self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
read_value.Evaluator(self.job),
))))
self.assertEqual(
{
'read_value_chromium@aaaaaaa_%s' % (attempt, ): {
'benchmark':
'some_benchmark',
'mode':
'graph_json',
'results_filename':
'some_benchmark/perf_results.json',
'histogram_options': {
'tir_label': None,
'story': None,
'statistic': None,
},
'graph_json_options': {
'chart': 'chart',
'trace': 'must_not_be_found',
},
'errors': [{
'reason': 'DependencyFailed',
'message': mock.ANY,
}],
'status':
'failed',
'tries':
1,
}
for attempt in range(10)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select', target_task=None,
payload={}),
evaluators.Selector(task_type='read_value')))
def __init__(self):
super(Serializer, self).__init__(predicate=evaluators.All(
evaluators.TaskTypeEq('read_value'),
evaluators.TaskStatusIn(
{'ongoing', 'failed', 'completed', 'cancelled'}),
),
delegate=ResultSerializer)
def testEvaluateFailedDependency(self, *_):
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(evaluators=(
functools.partial(FakeFindIsolateFailed, self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
# When we initiate the run_test tasks, we should immediately see the tasks
# failing because the dependency has a hard failure status.
self.assertEqual(
dict([('build_aaaaaaa', mock.ANY)] +
[('run_test_aaaaaaa_%s' % (attempt, ), {
'status': 'failed',
'errors': mock.ANY,
'dimensions': DIMENSIONS,
'extra_args': [],
'swarming_server': 'some_server',
}) for attempt in range(11)]),
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
def __init__(self, job):
super(Evaluator, self).__init__(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.All(evaluators.TaskTypeEq('run_test'), ),
delegate=evaluators.DispatchByEventTypeEvaluator({
'initiate':
evaluators.FilteringEvaluator(predicate=evaluators.Not(
evaluators.TaskStatusIn(
{'ongoing', 'failed', 'completed'})),
delegate=InitiateEvaluator(
job)),
# For updates, we want to ensure that the initiate evaluator
# has a chance to run on 'pending' tasks.
'update':
evaluators.SequenceEvaluator([
evaluators.FilteringEvaluator(
predicate=evaluators.Not(
evaluators.TaskStatusIn(
{'ongoing', 'failed', 'completed'})),
delegate=InitiateEvaluator(job)),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskStatusIn({'ongoing'}),
delegate=UpdateEvaluator(job)),
])
})),
evaluators.TaskPayloadLiftingEvaluator(),
))
def __init__(self):
super(Serializer, self).__init__(predicate=evaluators.All(
evaluators.TaskTypeEq('find_culprit'),
evaluators.TaskStatusIn(
{'ongoing', 'failed', 'completed', 'cancelled'}),
),
delegate=AnalysisSerializer)
def __init__(self, job):
super(Evaluator, self).__init__(
predicate=evaluators.All(evaluators.TaskTypeEq('read_value'),
evaluators.TaskStatusIn({'pending'})),
delegate=evaluators.SequenceEvaluator(
evaluators=(evaluators.TaskPayloadLiftingEvaluator(),
ReadValueEvaluator(job))))
def setUp(self):
super(EvaluatorTest, self).setUp()
self.maxDiff = None
self.job = job_module.Job.New((), ())
# Set up a common evaluator for all the test cases.
self.evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(bisection_test_util.FakeFoundIsolate(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('run_test'),
delegate=evaluators.SequenceEvaluator(evaluators=(
bisection_test_util.FakeSuccessfulRunTest(self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
read_value.Evaluator(self.job),
))
def CompoundEvaluatorForTesting(self, fake_evaluator):
return evaluators.SequenceEvaluator([
evaluators.FilteringEvaluator(
predicate=evaluators.All(evaluators.TaskTypeEq('read_value'),
evaluators.TaskStatusIn({'pending'})),
delegate=evaluators.SequenceEvaluator(
[fake_evaluator,
evaluators.TaskPayloadLiftingEvaluator()])),
evaluators.SequenceEvaluator([
performance_bisection.Evaluator(self.job),
evaluators.TaskPayloadLiftingEvaluator(
exclude_keys={'commits'})
]),
])
def __init__(self, job):
super(Evaluator, self).__init__(
evaluators=(
evaluators.TaskPayloadLiftingEvaluator(),
evaluators.FilteringEvaluator(
predicate=evaluators.All(
evaluators.TaskTypeEq('find_isolate'),
evaluators.TaskIsEventTarget(),
evaluators.Not(
evaluators.TaskStatusIn(
{'completed', 'failed', 'cancelled'})),
),
delegate=evaluators.DispatchByEventTypeEvaluator({
'initiate': InitiateEvaluator(job),
'update': UpdateEvaluator(job),
})),
))
def __init__(self, job):
super(Evaluator, self).__init__(evaluators=(
evaluators.TaskPayloadLiftingEvaluator(),
evaluators.FilteringEvaluator(
predicate=evaluators.All(
evaluators.TaskTypeEq('run_test'),
evaluators.TaskIsEventTarget(),
),
delegate=evaluators.DispatchByEventTypeEvaluator({
'initiate':
evaluators.FilteringEvaluator(predicate=evaluators.Not(
evaluators.TaskStatusIn(
{'ongoing', 'failed', 'completed'})),
delegate=InitiateEvaluator(
job)),
'update':
evaluators.FilteringEvaluator(
predicate=evaluators.TaskStatusIn({'ongoing'}),
delegate=UpdateEvaluator(job)),
})),
))
def testMissingDependencyInputs(self):
job = job_module.Job.New((), ())
task_module.PopulateTaskGraph(
job,
task_module.TaskGraph(
vertices=[
task_module.TaskVertex(id='build_aaaaaaa',
vertex_type='find_isolate',
payload={
'builder': 'Some Builder',
'target':
'telemetry_perf_tests',
'bucket': 'luci.bucket',
'change': {
'commits': [{
'repository':
'chromium',
'git_hash':
'aaaaaaa',
}]
}
}),
task_module.TaskVertex(id='run_test_aaaaaaa_0',
vertex_type='run_test',
payload={
'swarming_server':
'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
}),
],
edges=[
task_module.Dependency(from_='run_test_aaaaaaa_0',
to='build_aaaaaaa')
],
))
# This time we're fine, there should be no errors.
self.assertEqual({},
task_module.Evaluate(
job,
event_module.Event(type='validate',
target_task=None,
payload={}),
run_test.Validator()))
# Send an initiate message then catch that we've not provided the required
# payload in the task when it's ongoing.
self.assertEqual(
{
'build_aaaaaaa': mock.ANY,
'run_test_aaaaaaa_0': {
'errors': [{
'cause': 'MissingDependencyInputs',
'message': mock.ANY
}]
}
},
task_module.Evaluate(
job,
event_module.Event(type='initiate',
target_task=None,
payload={}),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(evaluators=(
functools.partial(FakeNotFoundIsolate, job),
evaluators.TaskPayloadLiftingEvaluator(),
)),
alternative=run_test.Validator()),
))
def __init__(self, job):
super(Evaluator, self).__init__(
predicate=evaluators.All(
evaluators.TaskTypeEq('find_culprit'),
evaluators.Not(evaluators.TaskStatusIn({'completed', 'failed'}))),
delegate=FindCulprit(job))
def __init__(self):
super(Serializer,
self).__init__(predicate=evaluators.TaskTypeEq('run_test'),
delegate=TestSerializer)
def __init__(self):
super(Validator,
self).__init__(predicate=evaluators.TaskTypeEq('run_test'),
delegate=ReportError)
def testEvaluateToCompletion(self, swarming_task_result,
swarming_tasks_new):
swarming_tasks_new.return_value = {'task_id': 'task id'}
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(functools.partial(FakeFoundIsolate, self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
self.assertNotEqual({},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
# Ensure that we've found all the 'run_test' tasks.
self.assertEqual(
{
'run_test_aaaaaaa_%s' % (attempt, ): {
'status': 'ongoing',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
},
'swarming_task_id': 'task id',
'tries': 1,
}
for attempt in range(11)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select',
target_task=None, payload={}),
Selector(task_type='run_test')))
# Ensure that we've actually made the calls to the Swarming service.
swarming_tasks_new.assert_called()
self.assertGreaterEqual(swarming_tasks_new.call_count, 10)
# Then we propagate an event for each of the run_test tasks in the graph.
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 0,
'failure': False,
'outputs_ref': {
'isolatedserver': 'output isolate server',
'isolated': 'output isolate hash',
},
'state': 'COMPLETED',
}
for attempt in range(11):
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(
type='update',
target_task='run_test_aaaaaaa_%s' % (attempt, ),
payload={}), evaluator), 'Attempt #%s' % (attempt, ))
# Ensure that we've polled the status of each of the tasks, and that we've
# marked the tasks completed.
self.assertEqual(
{
'run_test_aaaaaaa_%s' % (attempt, ): {
'status': 'completed',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
},
'swarming_task_result': {
'bot_id': mock.ANY,
'state': 'COMPLETED',
'failure': False,
},
'isolate_server': 'output isolate server',
'isolate_hash': 'output isolate hash',
'swarming_task_id': 'task id',
'tries': 1,
}
for attempt in range(11)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select',
target_task=None, payload={}),
Selector(task_type='run_test')))
# Ensure that we've actually made the calls to the Swarming service.
swarming_task_result.assert_called()
self.assertGreaterEqual(swarming_task_result.call_count, 10)
def testEvaluateHandleFailures_Hard(self, swarming_task_stdout,
swarming_task_result,
swarming_tasks_new):
swarming_tasks_new.return_value = {'task_id': 'task id'}
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(functools.partial(FakeFoundIsolate, self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
self.assertNotEqual({},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
# We set it up so that when we poll the swarming task, that we're going to
# get an error status. We're expecting that hard failures are detected.
swarming_task_stdout.return_value = {
'output':
"""Traceback (most recent call last):
File "../../testing/scripts/run_performance_tests.py", line 282, in <module>
sys.exit(main())
File "../../testing/scripts/run_performance_tests.py", line 226, in main
benchmarks = args.benchmark_names.split(',')
AttributeError: 'Namespace' object has no attribute 'benchmark_names'"""
}
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 1,
'failure': True,
'outputs_ref': {
'isolatedserver': 'output isolate server',
'isolated': 'output isolate hash',
},
'state': 'COMPLETED',
}
for attempt in range(11):
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(
type='update',
target_task='run_test_aaaaaaa_%s' % (attempt, ),
payload={
'kind': 'pubsub_message',
'action': 'poll'
}), evaluator), 'Attempt #%s' % (attempt, ))
self.assertEqual(
{
'run_test_aaaaaaa_%s' % (attempt, ): {
'status': 'failed',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'errors': mock.ANY,
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
},
'swarming_task_result': {
'bot_id': mock.ANY,
'state': 'COMPLETED',
'failure': True,
},
'isolate_server': 'output isolate server',
'isolate_hash': 'output isolate hash',
'swarming_task_id': 'task id',
'tries': 1,
}
for attempt in range(11)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select',
target_task=None, payload={}),
Selector(task_type='run_test')))
def testEvaluateHandleFailures_Expired(self, swarming_task_result,
swarming_tasks_new):
swarming_tasks_new.return_value = {'task_id': 'task id'}
evaluator = evaluators.SequenceEvaluator(evaluators=(
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(
evaluators=(functools.partial(FakeFoundIsolate, self.job),
evaluators.TaskPayloadLiftingEvaluator()))),
run_test.Evaluator(self.job),
))
self.assertNotEqual({},
task_module.Evaluate(
self.job,
event_module.Event(type='initiate',
target_task=None,
payload={}), evaluator))
swarming_task_result.return_value = {
'state': 'EXPIRED',
}
for attempt in range(11):
self.assertNotEqual(
{},
task_module.Evaluate(
self.job,
event_module.Event(
type='update',
target_task='run_test_aaaaaaa_%s' % (attempt, ),
payload={
'kind': 'pubsub_message',
'action': 'poll'
}), evaluator), 'Attempt #%s' % (attempt, ))
self.assertEqual(
{
'run_test_aaaaaaa_%s' % (attempt, ): {
'status': 'failed',
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'errors': [
{
'reason': 'SwarmingExpired',
'message': mock.ANY
},
],
'extra_args': [],
'swarming_request_body': {
'name': mock.ANY,
'user': mock.ANY,
'priority': mock.ANY,
'task_slices': mock.ANY,
'tags': mock.ANY,
'pubsub_auth_token': mock.ANY,
'pubsub_topic': mock.ANY,
'pubsub_userdata': mock.ANY,
},
'swarming_task_result': {
'state': 'EXPIRED',
},
'swarming_task_id': 'task id',
'tries': 1,
}
for attempt in range(11)
},
task_module.Evaluate(
self.job,
event_module.Event(type='select',
target_task=None, payload={}),
Selector(task_type='run_test')))
def __init__(self):
super(Serializer, self).__init__(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=BuildSerializer)