def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', description='Task A', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task( kind='test', label='b', attributes={}, task={'task': 'def'}, optimization={'skip-unless-has-relevant-tests': None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { 'a': { 'kind': 'test', 'label': 'a', 'description': 'Task A', 'attributes': { 'attr': 'a-task', 'kind': 'test' }, 'task': { 'taskdef': True }, 'dependencies': { 'edgelabel': 'b' }, 'soft_dependencies': [], 'optimization': None, }, 'b': { 'kind': 'test', 'label': 'b', 'description': '', 'attributes': { 'kind': 'test' }, 'task': { 'task': 'def' }, 'dependencies': {}, 'soft_dependencies': [], 'optimization': { 'skip-unless-has-relevant-tests': None }, } })
def get_filtered_taskgraph(self, taskgraph, tasksregex): from taskgraph.graph import Graph from taskgraph.taskgraph import TaskGraph """ This class method filters all the tasks on basis of a regular expression and returns a new TaskGraph object """ # return original taskgraph if no regular expression is passed if not tasksregex: return taskgraph named_links_dict = taskgraph.graph.named_links_dict() filteredtasks = {} filterededges = set() regexprogram = re.compile(tasksregex) for key in taskgraph.graph.visit_postorder(): task = taskgraph.tasks[key] if regexprogram.match(task.label): filteredtasks[key] = task for depname, dep in named_links_dict[key].iteritems(): if regexprogram.match(dep): filterededges.add((key, dep, depname)) filtered_taskgraph = TaskGraph( filteredtasks, Graph(set(filteredtasks), filterededges)) return filtered_taskgraph
def test_create_tasks_fails_if_create_fails(self, create_task): "creat_tasks fails if a single create_task call fails" tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) def fail(*args): print("UHOH") raise RuntimeError('oh noes!') create_task.side_effect = fail with self.assertRaises(RuntimeError): create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", )
def test_create_tasks(self): tasks = { "tid-a": Task(kind="test", label="a", attributes={}, task={"payload": "hello world"}), "tid-b": Task(kind="test", label="b", attributes={}, task={"payload": "hello world"}), } label_to_taskid = {"a": "tid-a", "b": "tid-b"} graph = Graph(nodes={"tid-a", "tid-b"}, edges={("tid-a", "tid-b", "edge")}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task["payload"], "hello world") self.assertEqual(task["schedulerId"], "domain-level-4") # make sure the dependencies exist, at least for depid in task.get("dependencies", []): if depid == "decisiontask": # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def test_create_tasks(self): tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), 'tid-b': Task(kind='test', label='b', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task['payload'], 'hello world') self.assertEqual(task['schedulerId'], 'domain-level-4') # make sure the dependencies exist, at least for depid in task.get('dependencies', []): if depid == 'decisiontask': # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def inner(tasks): label_to_taskid = {k: k + '-tid' for k in tasks} for label, task_id in label_to_taskid.iteritems(): tasks[label].task_id = task_id graph = Graph(nodes=set(tasks), edges=set()) taskgraph = TaskGraph(tasks, graph) return taskgraph, label_to_taskid
def make_taskgraph(self, tasks): label_to_taskid = {k: k + "-tid" for k in tasks} for label, task_id in label_to_taskid.items(): tasks[label].task_id = task_id graph = Graph(nodes=set(tasks), edges=set()) taskgraph = TaskGraph(tasks, graph) return taskgraph, label_to_taskid
def test_round_trip(self): graph = TaskGraph( tasks={ "a": Task( kind="fancy", label="a", description="Task A", attributes={}, dependencies={"prereq": "b"}, # must match edges, below optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def"}, ), "b": Task( kind="pre", label="b", attributes={}, dependencies={}, optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def2"}, ), }, graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}), ) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def default_matches(self, attributes, parameters): method = target_tasks.get_method('default') graph = TaskGraph(tasks={ 'a': Task(kind='build', label='a', attributes=attributes, task={}), }, graph=Graph(nodes={'a'}, edges=set())) return 'a' in method(graph, parameters, {})
def test_transitive_closure_disjoint_edges(self): "transitive closure of a disjoint graph keeps those edges" self.assertEqual(self.disjoint.transitive_closure(set(['3', 'β'])), Graph(set(['1', '2', '3', 'β', 'γ']), { ('2', '1', 'red'), ('3', '1', 'red'), ('3', '2', 'green'), ('β', 'γ', 'κόκκινο'), }))
def test_transitive_closure_trees(self): "transitive closure of a tree, at two non-root nodes, is the two subtrees" self.assertEqual(self.tree.transitive_closure(set(['b', 'c'])), Graph(set(['b', 'c', 'd', 'e', 'f', 'g']), { ('b', 'd', 'K'), ('b', 'e', 'K'), ('c', 'f', 'N'), ('c', 'g', 'N'), }))
def test_transitive_closure_multi_edges(self): "transitive closure of a tree with multiple edges between nodes keeps those edges" self.assertEqual(self.multi_edges.transitive_closure(set(['3'])), Graph(set(['1', '2', '3']), { ('2', '1', 'red'), ('2', '1', 'blue'), ('3', '1', 'red'), ('3', '2', 'blue'), ('3', '2', 'green'), }))
def tg(request): if not hasattr(request.module, 'TASKS'): pytest.fail("'tg' fixture used from a module that didn't define the TASKS variable") tasks = request.module.TASKS for task in tasks: task.setdefault('task', {}) task['task'].setdefault('tags', {}) tasks = {t['label']: Task(**t) for t in tasks} return TaskGraph(tasks, Graph(tasks.keys(), set()))
def default_matches(self, attributes, parameters): method = target_tasks.get_method("default") graph = TaskGraph( tasks={ "a": Task(kind="build", label="a", attributes=attributes, task={}), }, graph=Graph(nodes={"a"}, edges=set()), ) return "a" in method(graph, parameters, {})
def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task( kind='test', label='b', attributes={}, task={'task': 'def'}, optimizations=[['seta']], # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { 'a': { 'kind': 'test', 'label': 'a', 'attributes': { 'attr': 'a-task', 'kind': 'test' }, 'task': { 'taskdef': True }, 'dependencies': { 'edgelabel': 'b' }, 'optimizations': [], }, 'b': { 'kind': 'test', 'label': 'b', 'attributes': { 'kind': 'test' }, 'task': { 'task': 'def' }, 'dependencies': {}, 'optimizations': [['seta']], } })
def default_matches(self, run_on_projects, project): method = target_tasks.get_method('default') graph = TaskGraph(tasks={ 'a': Task(kind='build', label='a', attributes={'run_on_projects': run_on_projects}, task={}), }, graph=Graph(nodes={'a'}, edges=set())) parameters = {'project': project} return 'a' in method(graph, parameters)
def test_taskgraph_to_json(self): tasks = { "a": Task( kind="test", label="a", description="Task A", attributes={"attr": "a-task"}, task={"taskdef": True}, ), "b": Task( kind="test", label="b", attributes={}, task={"task": "def"}, optimization={"skip-unless-has-relevant-tests": None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={"first": "a"}, ), } graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { "a": { "kind": "test", "label": "a", "description": "Task A", "attributes": {"attr": "a-task", "kind": "test"}, "task": {"taskdef": True}, "dependencies": {"edgelabel": "b"}, "soft_dependencies": [], "if_dependencies": [], "optimization": None, }, "b": { "kind": "test", "label": "b", "description": "", "attributes": {"kind": "test"}, "task": {"task": "def"}, "dependencies": {}, "soft_dependencies": [], "if_dependencies": [], "optimization": {"skip-unless-has-relevant-tests": None}, }, }, )
def tg(request): if not hasattr(request.module, "TASKS"): pytest.fail( "'tg' fixture used from a module that didn't define the TASKS variable" ) tasks = request.module.TASKS for task in tasks: task.setdefault("task", {}) task["task"].setdefault("tags", {}) tasks = {t["label"]: Task(**t) for t in tasks} return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_transitive_closure_trees(self): "transitive closure of a tree, at two non-root nodes, is the two subtrees" self.assertEqual( self.tree.transitive_closure(set(["b", "c"])), Graph( set(["b", "c", "d", "e", "f", "g"]), { ("b", "d", "K"), ("b", "e", "K"), ("c", "f", "N"), ("c", "g", "N"), }, ), )
def make_task_graph(self): tasks = { "a": Task(kind=None, label="a", attributes={}, task={}), "b": Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}), "c": Task(kind=None, label="c", attributes={"run_on_projects": ["try"]}, task={}), } graph = Graph(nodes=set("abc"), edges=set()) return TaskGraph(tasks, graph)
def make_task_graph(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={'run_on_projects': ['try']}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) return TaskGraph(tasks, graph)
def test_transitive_closure_disjoint_edges(self): "transitive closure of a disjoint graph keeps those edges" self.assertEqual( self.disjoint.transitive_closure(set(["3", "β"])), Graph( set(["1", "2", "3", "β", "γ"]), { ("2", "1", "red"), ("3", "1", "red"), ("3", "2", "green"), ("β", "γ", "κόκκινο"), }, ), )
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" os.environ['TASK_ID'] = 'decisiontask' tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks(GRAPH_CONFIG, taskgraph, label_to_taskid, {'level': '4'}) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def test_transitive_closure_multi_edges(self): "transitive closure of a tree with multiple edges between nodes keeps those edges" self.assertEqual( self.multi_edges.transitive_closure(set(["3"])), Graph( set(["1", "2", "3"]), { ("2", "1", "red"), ("2", "1", "blue"), ("3", "1", "red"), ("3", "2", "blue"), ("3", "2", "green"), }, ), )
def test_try_tasks(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) tg = TaskGraph(tasks, graph) method = target_tasks.get_method('try_tasks') config = os.path.join(os.getcwd(), 'try_task_config.json') orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax try: try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax # no try specifier self.assertEqual(method(tg, {'message': ''}), ['b']) # try syntax only self.assertEqual(method(tg, {'message': 'try: me'}), ['b']) # try task config only with open(config, 'w') as fh: fh.write('["c"]') self.assertEqual(method(tg, {'message': ''}), ['c']) with open(config, 'w') as fh: fh.write('{"c": {}}') self.assertEqual(method(tg, {'message': ''}), ['c']) # both syntax and config self.assertEqual(set(method(tg, {'message': 'try: me'})), set(['b', 'c'])) finally: try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax if os.path.isfile(config): os.remove(config)
def test_round_trip(self): graph = TaskGraph(tasks={ 'a': Task( kind='fancy', label='a', description='Task A', attributes={}, dependencies={'prereq': 'b'}, # must match edges, below optimization={'skip-unless-has-relevant-tests': None}, task={'task': 'def'}), 'b': Task( kind='pre', label='b', attributes={}, dependencies={}, optimization={'skip-unless-has-relevant-tests': None}, task={'task': 'def2'}), }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def test_try_tasks(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) tg = TaskGraph(tasks, graph) method = target_tasks.get_method('try_tasks') params = { 'message': '', 'target_task_labels': [], } orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax try: try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax # no try specifier self.assertEqual(method(tg, params), ['b']) # try syntax only params['message'] = 'try: me' self.assertEqual(method(tg, params), ['b']) # try task config only params['message'] = '' params['target_task_labels'] = ['c'] self.assertEqual(method(tg, params), ['c']) # both syntax and config params['message'] = 'try: me' self.assertEqual(set(method(tg, params)), set(['b', 'c'])) finally: try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" tasks = { "tid-a": Task(kind="test", label="a", attributes={}, task={"payload": "hello world"}), } label_to_taskid = {"a": "tid-a"} graph = Graph(nodes={"tid-a"}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task.get("dependencies"), ["decisiontask"])
def test_round_trip(self): graph = TaskGraph( tasks={ 'a': Task( kind='fancy', label='a', attributes={}, dependencies={'prereq': 'b'}, # must match edges, below optimizations=[['seta']], task={'task': 'def'}), 'b': Task(kind='pre', label='b', attributes={}, dependencies={}, optimizations=[['seta']], task={'task': 'def2'}), }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task.get('dependencies'), ["decisiontask"])
def test_transitive_closure_empty(self): "transitive closure of an empty set is an empty graph" g = Graph(set(['a', 'b', 'c']), {('a', 'b', 'L'), ('a', 'c', 'L')}) self.assertEqual(g.transitive_closure(set()), Graph(set(), set()))
def test_transitive_closure_disjoint(self): "transitive closure of a disjoint set is a subset" g = Graph(set(['a', 'b', 'c']), set()) self.assertEqual(g.transitive_closure(set(['a', 'c'])), Graph(set(['a', 'c']), set()))