def test_taskgraph_to_json(self):
tasks = {
'a':
Task(kind='test',
label='a',
description='Task A',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b':
Task(
kind='test',
label='b',
attributes={},
task={'task': 'def'},
optimization={'skip-unless-has-relevant-tests': None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res, {
'a': {
'kind': 'test',
'label': 'a',
'description': 'Task A',
'attributes': {
'attr': 'a-task',
'kind': 'test'
},
'task': {
'taskdef': True
},
'dependencies': {
'edgelabel': 'b'
},
'soft_dependencies': [],
'optimization': None,
},
'b': {
'kind': 'test',
'label': 'b',
'description': '',
'attributes': {
'kind': 'test'
},
'task': {
'task': 'def'
},
'dependencies': {},
'soft_dependencies': [],
'optimization': {
'skip-unless-has-relevant-tests': None
},
}
})
def get_filtered_taskgraph(self, taskgraph, tasksregex):
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
"""
This class method filters all the tasks on basis of a regular expression
and returns a new TaskGraph object
"""
# return original taskgraph if no regular expression is passed
if not tasksregex:
return taskgraph
named_links_dict = taskgraph.graph.named_links_dict()
filteredtasks = {}
filterededges = set()
regexprogram = re.compile(tasksregex)
for key in taskgraph.graph.visit_postorder():
task = taskgraph.tasks[key]
if regexprogram.match(task.label):
filteredtasks[key] = task
for depname, dep in named_links_dict[key].iteritems():
if regexprogram.match(dep):
filterededges.add((key, dep, depname))
filtered_taskgraph = TaskGraph(
filteredtasks, Graph(set(filteredtasks), filterededges))
return filtered_taskgraph
def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError('oh noes!')
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
def test_create_tasks(self):
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
"tid-b":
Task(kind="test",
label="b",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a", "b": "tid-b"}
graph = Graph(nodes={"tid-a", "tid-b"},
edges={("tid-a", "tid-b", "edge")})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task["payload"], "hello world")
self.assertEqual(task["schedulerId"], "domain-level-4")
# make sure the dependencies exist, at least
for depid in task.get("dependencies", []):
if depid == "decisiontask":
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def test_create_tasks(self):
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
'tid-b':
Task(kind='test',
label='b',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
graph = Graph(nodes={'tid-a', 'tid-b'},
edges={('tid-a', 'tid-b', 'edge')})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task['payload'], 'hello world')
self.assertEqual(task['schedulerId'], 'domain-level-4')
# make sure the dependencies exist, at least
for depid in task.get('dependencies', []):
if depid == 'decisiontask':
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def inner(tasks):
label_to_taskid = {k: k + '-tid' for k in tasks}
for label, task_id in label_to_taskid.iteritems():
tasks[label].task_id = task_id
graph = Graph(nodes=set(tasks), edges=set())
taskgraph = TaskGraph(tasks, graph)
return taskgraph, label_to_taskid
def make_taskgraph(self, tasks):
label_to_taskid = {k: k + "-tid" for k in tasks}
for label, task_id in label_to_taskid.items():
tasks[label].task_id = task_id
graph = Graph(nodes=set(tasks), edges=set())
taskgraph = TaskGraph(tasks, graph)
return taskgraph, label_to_taskid
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a": Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b": Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def default_matches(self, attributes, parameters):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a':
Task(kind='build', label='a', attributes=attributes, task={}),
},
graph=Graph(nodes={'a'}, edges=set()))
return 'a' in method(graph, parameters, {})
def test_transitive_closure_disjoint_edges(self):
"transitive closure of a disjoint graph keeps those edges"
self.assertEqual(self.disjoint.transitive_closure(set(['3', 'β'])),
Graph(set(['1', '2', '3', 'β', 'γ']), {
('2', '1', 'red'),
('3', '1', 'red'),
('3', '2', 'green'),
('β', 'γ', 'κόκκινο'),
}))
def test_transitive_closure_trees(self):
"transitive closure of a tree, at two non-root nodes, is the two subtrees"
self.assertEqual(self.tree.transitive_closure(set(['b', 'c'])),
Graph(set(['b', 'c', 'd', 'e', 'f', 'g']), {
('b', 'd', 'K'),
('b', 'e', 'K'),
('c', 'f', 'N'),
('c', 'g', 'N'),
}))
def test_transitive_closure_multi_edges(self):
"transitive closure of a tree with multiple edges between nodes keeps those edges"
self.assertEqual(self.multi_edges.transitive_closure(set(['3'])),
Graph(set(['1', '2', '3']), {
('2', '1', 'red'),
('2', '1', 'blue'),
('3', '1', 'red'),
('3', '2', 'blue'),
('3', '2', 'green'),
}))
def tg(request):
if not hasattr(request.module, 'TASKS'):
pytest.fail("'tg' fixture used from a module that didn't define the TASKS variable")
tasks = request.module.TASKS
for task in tasks:
task.setdefault('task', {})
task['task'].setdefault('tags', {})
tasks = {t['label']: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def default_matches(self, attributes, parameters):
method = target_tasks.get_method("default")
graph = TaskGraph(
tasks={
"a": Task(kind="build",
label="a",
attributes=attributes,
task={}),
},
graph=Graph(nodes={"a"}, edges=set()),
)
return "a" in method(graph, parameters, {})
def test_taskgraph_to_json(self):
tasks = {
'a':
Task(kind='test',
label='a',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b':
Task(
kind='test',
label='b',
attributes={},
task={'task': 'def'},
optimizations=[['seta']],
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res, {
'a': {
'kind': 'test',
'label': 'a',
'attributes': {
'attr': 'a-task',
'kind': 'test'
},
'task': {
'taskdef': True
},
'dependencies': {
'edgelabel': 'b'
},
'optimizations': [],
},
'b': {
'kind': 'test',
'label': 'b',
'attributes': {
'kind': 'test'
},
'task': {
'task': 'def'
},
'dependencies': {},
'optimizations': [['seta']],
}
})
def default_matches(self, run_on_projects, project):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a':
Task(kind='build',
label='a',
attributes={'run_on_projects': run_on_projects},
task={}),
},
graph=Graph(nodes={'a'}, edges=set()))
parameters = {'project': project}
return 'a' in method(graph, parameters)
def test_taskgraph_to_json(self):
tasks = {
"a": Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b": Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {"attr": "a-task", "kind": "test"},
"task": {"taskdef": True},
"dependencies": {"edgelabel": "b"},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {"kind": "test"},
"task": {"task": "def"},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {"skip-unless-has-relevant-tests": None},
},
},
)
def tg(request):
if not hasattr(request.module, "TASKS"):
pytest.fail(
"'tg' fixture used from a module that didn't define the TASKS variable"
)
tasks = request.module.TASKS
for task in tasks:
task.setdefault("task", {})
task["task"].setdefault("tags", {})
tasks = {t["label"]: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_transitive_closure_trees(self):
"transitive closure of a tree, at two non-root nodes, is the two subtrees"
self.assertEqual(
self.tree.transitive_closure(set(["b", "c"])),
Graph(
set(["b", "c", "d", "e", "f", "g"]),
{
("b", "d", "K"),
("b", "e", "K"),
("c", "f", "N"),
("c", "g", "N"),
},
),
)
def make_task_graph(self):
tasks = {
"a":
Task(kind=None, label="a", attributes={}, task={}),
"b":
Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}),
"c":
Task(kind=None,
label="c",
attributes={"run_on_projects": ["try"]},
task={}),
}
graph = Graph(nodes=set("abc"), edges=set())
return TaskGraph(tasks, graph)
def make_task_graph(self):
tasks = {
'a':
Task(kind=None, label='a', attributes={}, task={}),
'b':
Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}),
'c':
Task(kind=None,
label='c',
attributes={'run_on_projects': ['try']},
task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
return TaskGraph(tasks, graph)
def test_transitive_closure_disjoint_edges(self):
"transitive closure of a disjoint graph keeps those edges"
self.assertEqual(
self.disjoint.transitive_closure(set(["3", "β"])),
Graph(
set(["1", "2", "3", "β", "γ"]),
{
("2", "1", "red"),
("3", "1", "red"),
("3", "2", "green"),
("β", "γ", "κόκκινο"),
},
),
)
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
os.environ['TASK_ID'] = 'decisiontask'
tasks = {
'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(GRAPH_CONFIG, taskgraph, label_to_taskid, {'level': '4'})
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def test_transitive_closure_multi_edges(self):
"transitive closure of a tree with multiple edges between nodes keeps those edges"
self.assertEqual(
self.multi_edges.transitive_closure(set(["3"])),
Graph(
set(["1", "2", "3"]),
{
("2", "1", "red"),
("2", "1", "blue"),
("3", "1", "red"),
("3", "2", "blue"),
("3", "2", "green"),
},
),
)
def test_try_tasks(self):
tasks = {
'a': Task(kind=None, label='a', attributes={}, task={}),
'b': Task(kind=None,
label='b',
attributes={'at-at': 'yep'},
task={}),
'c': Task(kind=None, label='c', attributes={}, task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
tg = TaskGraph(tasks, graph)
method = target_tasks.get_method('try_tasks')
config = os.path.join(os.getcwd(), 'try_task_config.json')
orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
try:
try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
# no try specifier
self.assertEqual(method(tg, {'message': ''}), ['b'])
# try syntax only
self.assertEqual(method(tg, {'message': 'try: me'}), ['b'])
# try task config only
with open(config, 'w') as fh:
fh.write('["c"]')
self.assertEqual(method(tg, {'message': ''}), ['c'])
with open(config, 'w') as fh:
fh.write('{"c": {}}')
self.assertEqual(method(tg, {'message': ''}), ['c'])
# both syntax and config
self.assertEqual(set(method(tg, {'message': 'try: me'})),
set(['b', 'c']))
finally:
try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
if os.path.isfile(config):
os.remove(config)
def test_round_trip(self):
graph = TaskGraph(tasks={
'a': Task(
kind='fancy',
label='a',
description='Task A',
attributes={},
dependencies={'prereq': 'b'}, # must match edges, below
optimization={'skip-unless-has-relevant-tests': None},
task={'task': 'def'}),
'b': Task(
kind='pre',
label='b',
attributes={},
dependencies={},
optimization={'skip-unless-has-relevant-tests': None},
task={'task': 'def2'}),
}, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def test_try_tasks(self):
tasks = {
'a': Task(kind=None, label='a', attributes={}, task={}),
'b': Task(kind=None,
label='b',
attributes={'at-at': 'yep'},
task={}),
'c': Task(kind=None, label='c', attributes={}, task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
tg = TaskGraph(tasks, graph)
method = target_tasks.get_method('try_tasks')
params = {
'message': '',
'target_task_labels': [],
}
orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
try:
try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
# no try specifier
self.assertEqual(method(tg, params), ['b'])
# try syntax only
params['message'] = 'try: me'
self.assertEqual(method(tg, params), ['b'])
# try task config only
params['message'] = ''
params['target_task_labels'] = ['c']
self.assertEqual(method(tg, params), ['c'])
# both syntax and config
params['message'] = 'try: me'
self.assertEqual(set(method(tg, params)), set(['b', 'c']))
finally:
try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a"}
graph = Graph(nodes={"tid-a"}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task.get("dependencies"), ["decisiontask"])
def test_round_trip(self):
graph = TaskGraph(
tasks={
'a':
Task(
kind='fancy',
label='a',
attributes={},
dependencies={'prereq': 'b'}, # must match edges, below
optimizations=[['seta']],
task={'task': 'def'}),
'b':
Task(kind='pre',
label='b',
attributes={},
dependencies={},
optimizations=[['seta']],
task={'task': 'def2'}),
},
graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task.get('dependencies'), ["decisiontask"])
def test_transitive_closure_empty(self):
"transitive closure of an empty set is an empty graph"
g = Graph(set(['a', 'b', 'c']), {('a', 'b', 'L'), ('a', 'c', 'L')})
self.assertEqual(g.transitive_closure(set()),
Graph(set(), set()))
def test_transitive_closure_disjoint(self):
"transitive closure of a disjoint set is a subset"
g = Graph(set(['a', 'b', 'c']), set())
self.assertEqual(g.transitive_closure(set(['a', 'c'])),
Graph(set(['a', 'c']), set()))
