def testPopulateAndEvaluateAdderGraph(self): job = job_module.Job.New((), ()) task_graph = task_module.TaskGraph( vertices=[ task_module.TaskVertex(id='input0', vertex_type='constant', payload={'value': 0}), task_module.TaskVertex(id='input1', vertex_type='constant', payload={'value': 1}), task_module.TaskVertex(id='plus', vertex_type='operator+', payload={}), ], edges=[ task_module.Dependency(from_='plus', to='input0'), task_module.Dependency(from_='plus', to='input1'), ], ) task_module.PopulateTaskGraph(job, task_graph) def AdderEvaluator(task, _, accumulator): if task.task_type == 'constant': accumulator[task.id] = task.payload.get('value', 0) elif task.task_type == 'operator+': inputs = [accumulator.get(dep) for dep in task.dependencies] accumulator[task.id] = functools.reduce( lambda a, v: a + v, inputs) accumulator = task_module.Evaluate(job, {}, AdderEvaluator) self.assertEqual(1, accumulator.get('plus'))
def testPopulateEvaluateCallCounts(self): job = job_module.Job.New((), ()) task_module.PopulateTaskGraph( job, task_module.TaskGraph(vertices=[ task_module.TaskVertex(id='leaf_0', vertex_type='node', payload={}), task_module.TaskVertex(id='leaf_1', vertex_type='node', payload={}), task_module.TaskVertex(id='parent', vertex_type='node', payload={}), ], edges=[ task_module.Dependency(from_='parent', to='leaf_0'), task_module.Dependency(from_='parent', to='leaf_1'), ])) calls = {} def CallCountEvaluator(task, event, accumulator): logging.debug('Evaluate(%s, %s, %s) called.', task.id, event, accumulator) calls[task.id] = calls.get(task.id, 0) + 1 return None task_module.Evaluate(job, 'test', CallCountEvaluator) self.assertDictEqual({ 'leaf_0': 1, 'leaf_1': 1, 'parent': 1, }, calls)
def testPopulateCycles(self): job = job_module.Job.New((), ()) task_graph = task_module.TaskGraph( vertices=[ task_module.TaskVertex(id='node_0', vertex_type='process', payload={}), task_module.TaskVertex(id='node_1', vertex_type='process', payload={}) ], edges=[ task_module.Dependency(from_='node_0', to='node_1'), task_module.Dependency(from_='node_1', to='node_0') ]) task_module.PopulateTaskGraph(job, task_graph) calls = {} def CycleEvaluator(task, event, accumulator): logging.debug('Evaluate(%s, %s, %s) called.', task.id, event, accumulator) calls[task.id] = calls.get(task.id, 0) + 1 return None task_module.Evaluate(job, 'test', CycleEvaluator) self.assertDictEqual({'node_0': 1, 'node_1': 1}, calls)
def CreateGraph(options): if not isinstance(options, TaskOptions): raise ValueError('options is not an instance of run_test.TaskOptions') subgraph = find_isolate.CreateGraph(options.build_options) find_isolate_tasks = [ task for task in subgraph.vertices if task.vertex_type == 'find_isolate' ] assert len(find_isolate_tasks) == 1 find_isolate_task = find_isolate_tasks[0] subgraph.vertices.extend([ task_module.TaskVertex(id=TaskId( find_isolate.ChangeId(options.build_options.change), attempt), vertex_type='run_test', payload={ 'swarming_server': options.swarming_server, 'dimensions': options.dimensions, 'extra_args': options.extra_args, 'change': options.build_options.change.AsDict(), 'index': attempt, }) for attempt in range(options.attempts) ]) subgraph.edges.extend([ task_module.Dependency(from_=task.id, to=find_isolate_task.id) for task in subgraph.vertices if task.vertex_type == 'run_test' ]) return subgraph
def GenerateVertexAndDep(attempts): for attempt in range(attempts): change_id = find_isolate.ChangeId( options.test_options.build_options.change) read_value_id = 'read_value_%s_%s' % (change_id, attempt) run_test_id = run_test.TaskId(change_id, attempt) yield (task_module.TaskVertex( id=read_value_id, vertex_type='read_value', payload={ 'benchmark': options.benchmark, 'mode': options.mode, 'results_filename': path, 'histogram_options': { 'grouping_label': options.histogram_options.grouping_label, 'story': options.histogram_options.story, 'statistic': options.histogram_options.statistic, }, 'graph_json_options': { 'chart': options.graph_json_options.chart, 'trace': options.graph_json_options.trace }, 'change': options.test_options.build_options.change.AsDict(), 'index': attempt, }), task_module.Dependency(from_=read_value_id, to=run_test_id))
def GraphExtender(_): task_module.ExtendTaskGraph(self.job, vertices=[], dependencies=[ task_module.Dependency( from_='unknown', to=task.id) ])
def __call__(self, accumulator): # Outline: # - Given the job and task, extend the TaskGraph to add new tasks and # dependencies, being careful to filter the IDs from what we already see # in the accumulator to avoid graph amendment errors. # - If we do encounter graph amendment errors, we should log those and not # block progress because that can only happen if there's concurrent # updates being performed with the same actions. build_option_template = BuildOptionTemplate( **self.task.payload.get('build_option_template')) test_option_template = TestOptionTemplate( **self.task.payload.get('test_option_template')) # The ReadOptionTemplate is special because it has nested structures, so # we'll have to reconstitute those accordingly. read_option_template_map = self.task.payload.get( 'read_option_template') read_option_template = ReadOptionTemplate( benchmark=self.task.payload.get('read_option_template').get( 'benchmark'), histogram_options=read_value.HistogramOptions( **read_option_template_map.get('histogram_options')), graph_json_options=read_value.GraphJsonOptions( **read_option_template_map.get('graph_json_options')), mode=read_option_template_map.get('mode')) analysis_options_dict = self.task.payload.get('analysis_options') if self.additional_attempts: analysis_options_dict['min_attempts'] = min( analysis_options_dict.get('min_attempts', 0) + self.additional_attempts, analysis_options_dict.get('max_attempts', 100)) analysis_options = AnalysisOptions(**analysis_options_dict) new_subgraph = read_value.CreateGraph( _CreateReadTaskOptions(build_option_template, test_option_template, read_option_template, analysis_options, self.change, self.task.payload.get('arguments', {}))) try: task_module.ExtendTaskGraph( self.job, vertices=[ # Add all of the new vertices we do not have in the graph yet. v for v in new_subgraph.vertices if v.id not in accumulator ], dependencies=[ # Only add dependencies to the new 'read_value' tasks. task_module.Dependency(from_=self.task.id, to=v.id) for v in new_subgraph.vertices if v.id not in accumulator and v.vertex_type == 'read_value' ]) except task_module.InvalidAmendment as e: logging.error('Failed to amend graph: %s', e)
def GraphExtender(_): task_module.ExtendTaskGraph( self.job, vertices=[ task_module.TaskVertex(id=task.id, vertex_type='duplicate', payload={}) ], dependencies=[ task_module.Dependency(from_='task_2', to=task.id) ])
def setUp(self): super(EvaluateTest, self).setUp() self.maxDiff = None # pylint: disable=invalid-name self.job = job_module.Job.New((), ()) task_module.PopulateTaskGraph( self.job, task_module.TaskGraph(vertices=[ task_module.TaskVertex(id='task_0', vertex_type='task', payload={}), task_module.TaskVertex(id='task_1', vertex_type='task', payload={}), task_module.TaskVertex(id='task_2', vertex_type='task', payload={}), ], edges=[ task_module.Dependency(from_='task_2', to='task_0'), task_module.Dependency(from_='task_2', to='task_1'), ]))
def GraphExtender(_): logging.debug('New revisions: %s', new_positions) task_module.ExtendTaskGraph(job, [ task_module.TaskVertex(id='rev_%s' % (rev, ), vertex_type='revision', payload={ 'revision': '%s' % (rev, ), 'position': rev }) for rev in new_positions ], [ task_module.Dependency(from_='bisection', to='rev_%s' % (rev, )) for rev in new_positions ])
def setUp(self): super(EvaluatorTest, self).setUp() self.maxDiff = None self.job = job_module.Job.New((), ()) task_module.PopulateTaskGraph( self.job, task_module.TaskGraph( vertices=[ task_module.TaskVertex( id='build_aaaaaaa', vertex_type='find_isolate', payload={ 'builder': 'Some Builder', 'target': 'telemetry_perf_tests', 'bucket': 'luci.bucket', 'change': { 'commits': [{ 'repository': 'chromium', 'git_hash': 'aaaaaaa', }] } }) ] + [ task_module.TaskVertex( id='run_test_aaaaaaa_%s' % (attempt, ), vertex_type='run_test', payload={ 'swarming_server': 'some_server', 'dimensions': DIMENSIONS, 'extra_args': [], }) for attempt in range(11) ], edges=[ task_module.Dependency( from_='run_test_aaaaaaa_%s' % (attempt, ), to='build_aaaaaaa') for attempt in range(11) ], ))
def CreateGraph(options, arguments=None): if not isinstance(options, TaskOptions): raise ValueError( 'options must be an instance of performance_bisection.TaskOptions') start_change = options.start_change end_change = options.end_change if options.pinned_change: start_change.Update(options.pinned_change) end_change.Update(options.pinned_change) start_change_read_options = _CreateReadTaskOptions( options.build_option_template, options.test_option_template, options.read_option_template, options.analysis_options, start_change, arguments if arguments else {}) end_change_read_options = _CreateReadTaskOptions( options.build_option_template, options.test_option_template, options.read_option_template, options.analysis_options, end_change, arguments if arguments else {}) # Given the start_change and end_change, we create two subgraphs that we # depend on from the 'find_culprit' task. This means we'll need to create # independent test options and build options from the template provided by the # caller. start_subgraph = read_value.CreateGraph(start_change_read_options) end_subgraph = read_value.CreateGraph(end_change_read_options) # Then we add a dependency from the 'FindCulprit' task with the payload # describing the options set for the performance bisection. find_culprit_task = task_module.TaskVertex( id='performance_bisection', vertex_type='find_culprit', payload={ 'start_change': options.start_change.AsDict(), 'end_change': options.end_change.AsDict(), 'pinned_change': options.pinned_change.AsDict() if options.pinned_change else None, # We still persist the templates, because we'll need that data in case # we are going to extend the graph with the same build/test templates # in subgraphs. 'analysis_options': options.analysis_options._asdict(), 'build_option_template': options.build_option_template._asdict(), 'test_option_template': options.test_option_template._asdict(), 'read_option_template': { 'histogram_options': options.read_option_template.histogram_options._asdict(), 'graph_json_options': options.read_option_template.graph_json_options._asdict(), 'benchmark': options.read_option_template.benchmark, 'mode': options.read_option_template.mode, }, # Because this is a performance bisection, we'll hard-code the # comparison mode as 'performance'. 'comparison_mode': 'performance', 'arguments': arguments if arguments else {}, }) return task_module.TaskGraph( vertices=list( itertools.chain(start_subgraph.vertices, end_subgraph.vertices)) + [find_culprit_task], edges=list(itertools.chain(start_subgraph.edges, end_subgraph.edges)) + [ task_module.Dependency(from_=find_culprit_task.id, to=v.id) for v in itertools.chain(start_subgraph.vertices, end_subgraph.vertices) if v.vertex_type == 'read_value' ])
def testPouplateAndEvaluateGrowingGraph(self): job = job_module.Job.New((), ()) task_module.PopulateTaskGraph( job, task_module.TaskGraph(vertices=[ task_module.TaskVertex(id='rev_0', vertex_type='revision', payload={ 'revision': '0', 'position': 0 }), task_module.TaskVertex(id='rev_100', vertex_type='revision', payload={ 'revision': '100', 'position': 100 }), task_module.TaskVertex(id='bisection', vertex_type='bisection', payload={}), ], edges=[ task_module.Dependency(from_='bisection', to='rev_0'), task_module.Dependency(from_='bisection', to='rev_100'), ])) def FindMidpoint(a, b): offset = (b - a) // 2 if offset == 0: return None return a + offset def ExplorationEvaluator(task, event, accumulator): logging.debug('Evaluating: %s, %s, %s', task, event, accumulator) if task.task_type == 'revision': accumulator[task.id] = task.payload return if task.task_type == 'bisection': rev_positions = list( sorted( accumulator.get(dep).get('position') for dep in task.dependencies)) results = list(rev_positions) insertion_list = exploration.Speculate( rev_positions, # Assume we always find a difference between two positions. lambda *_: True, # Do nothing when we encounter an unknown error. lambda _: None, # Provide the function that will find the midpoint between two # revisions. FindMidpoint, # Speculate two levels deep in the bisection space. levels=2) for index, change in insertion_list: results.insert(index, change) new_positions = set(results) - set(rev_positions) if new_positions: def GraphExtender(_): logging.debug('New revisions: %s', new_positions) task_module.ExtendTaskGraph(job, [ task_module.TaskVertex(id='rev_%s' % (rev, ), vertex_type='revision', payload={ 'revision': '%s' % (rev, ), 'position': rev }) for rev in new_positions ], [ task_module.Dependency(from_='bisection', to='rev_%s' % (rev, )) for rev in new_positions ]) return [GraphExtender] accumulator = task_module.Evaluate(job, None, ExplorationEvaluator) self.assertEqual(list(sorted(accumulator)), sorted(['rev_%s' % (rev, ) for rev in range(0, 101)]))
def testMissingDependencyInputs(self): job = job_module.Job.New((), ()) task_module.PopulateTaskGraph( job, task_module.TaskGraph( vertices=[ task_module.TaskVertex(id='build_aaaaaaa', vertex_type='find_isolate', payload={ 'builder': 'Some Builder', 'target': 'telemetry_perf_tests', 'bucket': 'luci.bucket', 'change': { 'commits': [{ 'repository': 'chromium', 'git_hash': 'aaaaaaa', }] } }), task_module.TaskVertex(id='run_test_aaaaaaa_0', vertex_type='run_test', payload={ 'swarming_server': 'some_server', 'dimensions': DIMENSIONS, 'extra_args': [], }), ], edges=[ task_module.Dependency(from_='run_test_aaaaaaa_0', to='build_aaaaaaa') ], )) # This time we're fine, there should be no errors. self.assertEqual({}, task_module.Evaluate( job, event_module.Event(type='validate', target_task=None, payload={}), run_test.Validator())) # Send an initiate message then catch that we've not provided the required # payload in the task when it's ongoing. self.assertEqual( { 'build_aaaaaaa': mock.ANY, 'run_test_aaaaaaa_0': { 'errors': [{ 'cause': 'MissingDependencyInputs', 'message': mock.ANY }] } }, task_module.Evaluate( job, event_module.Event(type='initiate', target_task=None, payload={}), evaluators.FilteringEvaluator( predicate=evaluators.TaskTypeEq('find_isolate'), delegate=evaluators.SequenceEvaluator(evaluators=( functools.partial(FakeNotFoundIsolate, job), evaluators.TaskPayloadLiftingEvaluator(), )), alternative=run_test.Validator()), ))