def testPopulateAndEvaluateAdderGraph(self):
job = job_module.Job.New((), ())
task_graph = task_module.TaskGraph(
vertices=[
task_module.TaskVertex(id='input0',
vertex_type='constant',
payload={'value': 0}),
task_module.TaskVertex(id='input1',
vertex_type='constant',
payload={'value': 1}),
task_module.TaskVertex(id='plus',
vertex_type='operator+',
payload={}),
],
edges=[
task_module.Dependency(from_='plus', to='input0'),
task_module.Dependency(from_='plus', to='input1'),
],
)
task_module.PopulateTaskGraph(job, task_graph)
def AdderEvaluator(task, _, accumulator):
if task.task_type == 'constant':
accumulator[task.id] = task.payload.get('value', 0)
elif task.task_type == 'operator+':
inputs = [accumulator.get(dep) for dep in task.dependencies]
accumulator[task.id] = functools.reduce(
lambda a, v: a + v, inputs)
accumulator = task_module.Evaluate(job, {}, AdderEvaluator)
self.assertEqual(1, accumulator.get('plus'))
def testPopulateEvaluateCallCounts(self):
job = job_module.Job.New((), ())
task_module.PopulateTaskGraph(
job,
task_module.TaskGraph(vertices=[
task_module.TaskVertex(id='leaf_0',
vertex_type='node',
payload={}),
task_module.TaskVertex(id='leaf_1',
vertex_type='node',
payload={}),
task_module.TaskVertex(id='parent',
vertex_type='node',
payload={}),
],
edges=[
task_module.Dependency(from_='parent',
to='leaf_0'),
task_module.Dependency(from_='parent',
to='leaf_1'),
]))
calls = {}
def CallCountEvaluator(task, event, accumulator):
logging.debug('Evaluate(%s, %s, %s) called.', task.id, event,
accumulator)
calls[task.id] = calls.get(task.id, 0) + 1
return None
task_module.Evaluate(job, 'test', CallCountEvaluator)
self.assertDictEqual({
'leaf_0': 1,
'leaf_1': 1,
'parent': 1,
}, calls)
def testPopulateCycles(self):
job = job_module.Job.New((), ())
task_graph = task_module.TaskGraph(
vertices=[
task_module.TaskVertex(id='node_0',
vertex_type='process',
payload={}),
task_module.TaskVertex(id='node_1',
vertex_type='process',
payload={})
],
edges=[
task_module.Dependency(from_='node_0', to='node_1'),
task_module.Dependency(from_='node_1', to='node_0')
])
task_module.PopulateTaskGraph(job, task_graph)
calls = {}
def CycleEvaluator(task, event, accumulator):
logging.debug('Evaluate(%s, %s, %s) called.', task.id, event,
accumulator)
calls[task.id] = calls.get(task.id, 0) + 1
return None
task_module.Evaluate(job, 'test', CycleEvaluator)
self.assertDictEqual({'node_0': 1, 'node_1': 1}, calls)
def CreateGraph(options):
if not isinstance(options, TaskOptions):
raise ValueError('options is not an instance of run_test.TaskOptions')
subgraph = find_isolate.CreateGraph(options.build_options)
find_isolate_tasks = [
task for task in subgraph.vertices
if task.vertex_type == 'find_isolate'
]
assert len(find_isolate_tasks) == 1
find_isolate_task = find_isolate_tasks[0]
subgraph.vertices.extend([
task_module.TaskVertex(id=TaskId(
find_isolate.ChangeId(options.build_options.change), attempt),
vertex_type='run_test',
payload={
'swarming_server': options.swarming_server,
'dimensions': options.dimensions,
'extra_args': options.extra_args,
'change':
options.build_options.change.AsDict(),
'index': attempt,
}) for attempt in range(options.attempts)
])
subgraph.edges.extend([
task_module.Dependency(from_=task.id, to=find_isolate_task.id)
for task in subgraph.vertices if task.vertex_type == 'run_test'
])
return subgraph
def GenerateVertexAndDep(attempts):
for attempt in range(attempts):
change_id = find_isolate.ChangeId(
options.test_options.build_options.change)
read_value_id = 'read_value_%s_%s' % (change_id, attempt)
run_test_id = run_test.TaskId(change_id, attempt)
yield (task_module.TaskVertex(
id=read_value_id,
vertex_type='read_value',
payload={
'benchmark': options.benchmark,
'mode': options.mode,
'results_filename': path,
'histogram_options': {
'grouping_label':
options.histogram_options.grouping_label,
'story': options.histogram_options.story,
'statistic': options.histogram_options.statistic,
},
'graph_json_options': {
'chart': options.graph_json_options.chart,
'trace': options.graph_json_options.trace
},
'change':
options.test_options.build_options.change.AsDict(),
'index': attempt,
}), task_module.Dependency(from_=read_value_id,
to=run_test_id))
def GraphExtender(_):
task_module.ExtendTaskGraph(self.job,
vertices=[],
dependencies=[
task_module.Dependency(
from_='unknown',
to=task.id)
])
def __call__(self, accumulator):
# Outline:
# - Given the job and task, extend the TaskGraph to add new tasks and
# dependencies, being careful to filter the IDs from what we already see
# in the accumulator to avoid graph amendment errors.
# - If we do encounter graph amendment errors, we should log those and not
# block progress because that can only happen if there's concurrent
# updates being performed with the same actions.
build_option_template = BuildOptionTemplate(
**self.task.payload.get('build_option_template'))
test_option_template = TestOptionTemplate(
**self.task.payload.get('test_option_template'))
# The ReadOptionTemplate is special because it has nested structures, so
# we'll have to reconstitute those accordingly.
read_option_template_map = self.task.payload.get(
'read_option_template')
read_option_template = ReadOptionTemplate(
benchmark=self.task.payload.get('read_option_template').get(
'benchmark'),
histogram_options=read_value.HistogramOptions(
**read_option_template_map.get('histogram_options')),
graph_json_options=read_value.GraphJsonOptions(
**read_option_template_map.get('graph_json_options')),
mode=read_option_template_map.get('mode'))
analysis_options_dict = self.task.payload.get('analysis_options')
if self.additional_attempts:
analysis_options_dict['min_attempts'] = min(
analysis_options_dict.get('min_attempts', 0) +
self.additional_attempts,
analysis_options_dict.get('max_attempts', 100))
analysis_options = AnalysisOptions(**analysis_options_dict)
new_subgraph = read_value.CreateGraph(
_CreateReadTaskOptions(build_option_template, test_option_template,
read_option_template, analysis_options,
self.change,
self.task.payload.get('arguments', {})))
try:
task_module.ExtendTaskGraph(
self.job,
vertices=[
# Add all of the new vertices we do not have in the graph yet.
v for v in new_subgraph.vertices if v.id not in accumulator
],
dependencies=[
# Only add dependencies to the new 'read_value' tasks.
task_module.Dependency(from_=self.task.id, to=v.id)
for v in new_subgraph.vertices if v.id not in accumulator
and v.vertex_type == 'read_value'
])
except task_module.InvalidAmendment as e:
logging.error('Failed to amend graph: %s', e)
def GraphExtender(_):
task_module.ExtendTaskGraph(
self.job,
vertices=[
task_module.TaskVertex(id=task.id,
vertex_type='duplicate',
payload={})
],
dependencies=[
task_module.Dependency(from_='task_2',
to=task.id)
])
def setUp(self):
super(EvaluateTest, self).setUp()
self.maxDiff = None # pylint: disable=invalid-name
self.job = job_module.Job.New((), ())
task_module.PopulateTaskGraph(
self.job,
task_module.TaskGraph(vertices=[
task_module.TaskVertex(id='task_0',
vertex_type='task',
payload={}),
task_module.TaskVertex(id='task_1',
vertex_type='task',
payload={}),
task_module.TaskVertex(id='task_2',
vertex_type='task',
payload={}),
],
edges=[
task_module.Dependency(from_='task_2',
to='task_0'),
task_module.Dependency(from_='task_2',
to='task_1'),
]))
def GraphExtender(_):
logging.debug('New revisions: %s', new_positions)
task_module.ExtendTaskGraph(job, [
task_module.TaskVertex(id='rev_%s' % (rev, ),
vertex_type='revision',
payload={
'revision': '%s' %
(rev, ),
'position': rev
}) for rev in new_positions
], [
task_module.Dependency(from_='bisection',
to='rev_%s' % (rev, ))
for rev in new_positions
])
def setUp(self):
super(EvaluatorTest, self).setUp()
self.maxDiff = None
self.job = job_module.Job.New((), ())
task_module.PopulateTaskGraph(
self.job,
task_module.TaskGraph(
vertices=[
task_module.TaskVertex(
id='build_aaaaaaa',
vertex_type='find_isolate',
payload={
'builder': 'Some Builder',
'target': 'telemetry_perf_tests',
'bucket': 'luci.bucket',
'change': {
'commits': [{
'repository': 'chromium',
'git_hash': 'aaaaaaa',
}]
}
})
] + [
task_module.TaskVertex(
id='run_test_aaaaaaa_%s' % (attempt, ),
vertex_type='run_test',
payload={
'swarming_server': 'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
}) for attempt in range(11)
],
edges=[
task_module.Dependency(
from_='run_test_aaaaaaa_%s' % (attempt, ),
to='build_aaaaaaa') for attempt in range(11)
],
))
def CreateGraph(options, arguments=None):
if not isinstance(options, TaskOptions):
raise ValueError(
'options must be an instance of performance_bisection.TaskOptions')
start_change = options.start_change
end_change = options.end_change
if options.pinned_change:
start_change.Update(options.pinned_change)
end_change.Update(options.pinned_change)
start_change_read_options = _CreateReadTaskOptions(
options.build_option_template, options.test_option_template,
options.read_option_template, options.analysis_options, start_change,
arguments if arguments else {})
end_change_read_options = _CreateReadTaskOptions(
options.build_option_template, options.test_option_template,
options.read_option_template, options.analysis_options, end_change,
arguments if arguments else {})
# Given the start_change and end_change, we create two subgraphs that we
# depend on from the 'find_culprit' task. This means we'll need to create
# independent test options and build options from the template provided by the
# caller.
start_subgraph = read_value.CreateGraph(start_change_read_options)
end_subgraph = read_value.CreateGraph(end_change_read_options)
# Then we add a dependency from the 'FindCulprit' task with the payload
# describing the options set for the performance bisection.
find_culprit_task = task_module.TaskVertex(
id='performance_bisection',
vertex_type='find_culprit',
payload={
'start_change':
options.start_change.AsDict(),
'end_change':
options.end_change.AsDict(),
'pinned_change':
options.pinned_change.AsDict() if options.pinned_change else None,
# We still persist the templates, because we'll need that data in case
# we are going to extend the graph with the same build/test templates
# in subgraphs.
'analysis_options':
options.analysis_options._asdict(),
'build_option_template':
options.build_option_template._asdict(),
'test_option_template':
options.test_option_template._asdict(),
'read_option_template': {
'histogram_options':
options.read_option_template.histogram_options._asdict(),
'graph_json_options':
options.read_option_template.graph_json_options._asdict(),
'benchmark':
options.read_option_template.benchmark,
'mode':
options.read_option_template.mode,
},
# Because this is a performance bisection, we'll hard-code the
# comparison mode as 'performance'.
'comparison_mode':
'performance',
'arguments':
arguments if arguments else {},
})
return task_module.TaskGraph(
vertices=list(
itertools.chain(start_subgraph.vertices, end_subgraph.vertices)) +
[find_culprit_task],
edges=list(itertools.chain(start_subgraph.edges, end_subgraph.edges)) + [
task_module.Dependency(from_=find_culprit_task.id, to=v.id)
for v in itertools.chain(start_subgraph.vertices,
end_subgraph.vertices)
if v.vertex_type == 'read_value'
])
def testPouplateAndEvaluateGrowingGraph(self):
job = job_module.Job.New((), ())
task_module.PopulateTaskGraph(
job,
task_module.TaskGraph(vertices=[
task_module.TaskVertex(id='rev_0',
vertex_type='revision',
payload={
'revision': '0',
'position': 0
}),
task_module.TaskVertex(id='rev_100',
vertex_type='revision',
payload={
'revision': '100',
'position': 100
}),
task_module.TaskVertex(id='bisection',
vertex_type='bisection',
payload={}),
],
edges=[
task_module.Dependency(from_='bisection',
to='rev_0'),
task_module.Dependency(from_='bisection',
to='rev_100'),
]))
def FindMidpoint(a, b):
offset = (b - a) // 2
if offset == 0:
return None
return a + offset
def ExplorationEvaluator(task, event, accumulator):
logging.debug('Evaluating: %s, %s, %s', task, event, accumulator)
if task.task_type == 'revision':
accumulator[task.id] = task.payload
return
if task.task_type == 'bisection':
rev_positions = list(
sorted(
accumulator.get(dep).get('position')
for dep in task.dependencies))
results = list(rev_positions)
insertion_list = exploration.Speculate(
rev_positions,
# Assume we always find a difference between two positions.
lambda *_: True,
# Do nothing when we encounter an unknown error.
lambda _: None,
# Provide the function that will find the midpoint between two
# revisions.
FindMidpoint,
# Speculate two levels deep in the bisection space.
levels=2)
for index, change in insertion_list:
results.insert(index, change)
new_positions = set(results) - set(rev_positions)
if new_positions:
def GraphExtender(_):
logging.debug('New revisions: %s', new_positions)
task_module.ExtendTaskGraph(job, [
task_module.TaskVertex(id='rev_%s' % (rev, ),
vertex_type='revision',
payload={
'revision': '%s' %
(rev, ),
'position': rev
}) for rev in new_positions
], [
task_module.Dependency(from_='bisection',
to='rev_%s' % (rev, ))
for rev in new_positions
])
return [GraphExtender]
accumulator = task_module.Evaluate(job, None, ExplorationEvaluator)
self.assertEqual(list(sorted(accumulator)),
sorted(['rev_%s' % (rev, ) for rev in range(0, 101)]))
def testMissingDependencyInputs(self):
job = job_module.Job.New((), ())
task_module.PopulateTaskGraph(
job,
task_module.TaskGraph(
vertices=[
task_module.TaskVertex(id='build_aaaaaaa',
vertex_type='find_isolate',
payload={
'builder': 'Some Builder',
'target':
'telemetry_perf_tests',
'bucket': 'luci.bucket',
'change': {
'commits': [{
'repository':
'chromium',
'git_hash':
'aaaaaaa',
}]
}
}),
task_module.TaskVertex(id='run_test_aaaaaaa_0',
vertex_type='run_test',
payload={
'swarming_server':
'some_server',
'dimensions': DIMENSIONS,
'extra_args': [],
}),
],
edges=[
task_module.Dependency(from_='run_test_aaaaaaa_0',
to='build_aaaaaaa')
],
))
# This time we're fine, there should be no errors.
self.assertEqual({},
task_module.Evaluate(
job,
event_module.Event(type='validate',
target_task=None,
payload={}),
run_test.Validator()))
# Send an initiate message then catch that we've not provided the required
# payload in the task when it's ongoing.
self.assertEqual(
{
'build_aaaaaaa': mock.ANY,
'run_test_aaaaaaa_0': {
'errors': [{
'cause': 'MissingDependencyInputs',
'message': mock.ANY
}]
}
},
task_module.Evaluate(
job,
event_module.Event(type='initiate',
target_task=None,
payload={}),
evaluators.FilteringEvaluator(
predicate=evaluators.TaskTypeEq('find_isolate'),
delegate=evaluators.SequenceEvaluator(evaluators=(
functools.partial(FakeNotFoundIsolate, job),
evaluators.TaskPayloadLiftingEvaluator(),
)),
alternative=run_test.Validator()),
))
