def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a"}
graph = Graph(nodes={"tid-a"}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError("oh noes!")
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a":
Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b":
Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def test_create_tasks(self):
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
"tid-b":
Task(kind="test",
label="b",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a", "b": "tid-b"}
graph = Graph(nodes={"tid-a", "tid-b"},
edges={("tid-a", "tid-b", "edge")})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task["payload"], "hello world")
self.assertEqual(task["schedulerId"], "domain-level-4")
# make sure the dependencies exist, at least
for depid in task.get("dependencies", []):
if depid == "decisiontask":
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def inner(tasks):
label_to_taskid = {k: k + "-tid" for k in tasks}
for label, task_id in label_to_taskid.items():
tasks[label].task_id = task_id
graph = Graph(nodes=set(tasks), edges=set())
taskgraph = TaskGraph(tasks, graph)
return taskgraph, label_to_taskid
def get_filtered_taskgraph(taskgraph, tasksregex):
"""
Filter all the tasks on basis of a regular expression
and returns a new TaskGraph object
"""
from gecko_taskgraph.graph import Graph
from gecko_taskgraph.taskgraph import TaskGraph
# return original taskgraph if no regular expression is passed
if not tasksregex:
return taskgraph
named_links_dict = taskgraph.graph.named_links_dict()
filteredtasks = {}
filterededges = set()
regexprogram = re.compile(tasksregex)
for key in taskgraph.graph.visit_postorder():
task = taskgraph.tasks[key]
if regexprogram.match(task.label):
filteredtasks[key] = task
for depname, dep in named_links_dict[key].items():
if regexprogram.match(dep):
filterededges.add((key, dep, depname))
filtered_taskgraph = TaskGraph(
filteredtasks, Graph(set(filteredtasks), filterededges)
)
return filtered_taskgraph
def default_matches(self, attributes, parameters):
method = target_tasks.get_method("default")
graph = TaskGraph(
tasks={
"a": Task(kind="build",
label="a",
attributes=attributes,
task={}),
},
graph=Graph(nodes={"a"}, edges=set()),
)
return "a" in method(graph, parameters, {})
def tg(request):
if not hasattr(request.module, "TASKS"):
pytest.fail(
"'tg' fixture used from a module that didn't define the TASKS variable"
)
tasks = request.module.TASKS
for task in tasks:
task.setdefault("task", {})
task["task"].setdefault("tags", {})
tasks = {t["label"]: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_transitive_closure_trees(self):
"transitive closure of a tree, at two non-root nodes, is the two subtrees"
self.assertEqual(
self.tree.transitive_closure({"b", "c"}),
Graph(
{"b", "c", "d", "e", "f", "g"},
{
("b", "d", "K"),
("b", "e", "K"),
("c", "f", "N"),
("c", "g", "N"),
},
),
)
def make_task_graph(self):
tasks = {
"a":
Task(kind=None, label="a", attributes={}, task={}),
"b":
Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}),
"c":
Task(kind=None,
label="c",
attributes={"run_on_projects": ["try"]},
task={}),
}
graph = Graph(nodes=set("abc"), edges=set())
return TaskGraph(tasks, graph)
def test_transitive_closure_disjoint_edges(self):
"transitive closure of a disjoint graph keeps those edges"
self.assertEqual(
self.disjoint.transitive_closure({"3", "β"}),
Graph(
{"1", "2", "3", "β", "γ"},
{
("2", "1", "red"),
("3", "1", "red"),
("3", "2", "green"),
("β", "γ", "κόκκινο"),
},
),
)
def test_transitive_closure_multi_edges(self):
"transitive closure of a tree with multiple edges between nodes keeps those edges"
self.assertEqual(
self.multi_edges.transitive_closure({"3"}),
Graph(
{"1", "2", "3"},
{
("2", "1", "red"),
("2", "1", "blue"),
("3", "1", "red"),
("3", "2", "blue"),
("3", "2", "green"),
},
),
)
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a"}
graph = Graph(nodes={"tid-a"}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task.get("dependencies"), ["decisiontask"])
RIDEALONG_BUILDS = {
"linux": ["linux-ridealong"],
"linux64": ["linux64-ridealong"],
}
GRAPH_CONFIG = {
"try": {"ridealong-builds": RIDEALONG_BUILDS},
}
for r in RIDEALONG_BUILDS.values():
tasks.update({k: v for k, v in [unittest_task(n + "-test", n) for n in r]})
unittest_tasks = {k: v for k, v in tasks.items() if "unittest_try_name" in v.attributes}
talos_tasks = {k: v for k, v in tasks.items() if "talos_try_name" in v.attributes}
graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
class TestTryOptionSyntax(unittest.TestCase):
def test_unknown_args(self):
"unknown arguments are ignored"
parameters = parse_message("try: --doubledash -z extra")
tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
# equilvant to "try:"..
self.assertEqual(tos.build_types, [])
self.assertEqual(tos.jobs, [])
def test_apostrophe_in_message(self):
"apostrophe does not break parsing"
parameters = parse_message("Increase spammy log's log level. try: -b do")
tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
def test_taskgraph_to_json(self):
tasks = {
"a":
Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b":
Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {
"attr": "a-task",
"kind": "test"
},
"task": {
"taskdef": True
},
"dependencies": {
"edgelabel": "b"
},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {
"kind": "test"
},
"task": {
"task": "def"
},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {
"skip-unless-has-relevant-tests": None
},
},
},
)
class TestTaskGraph(unittest.TestCase):
maxDiff = None
def test_taskgraph_to_json(self):
tasks = {
"a":
Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b":
Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {
"attr": "a-task",
"kind": "test"
},
"task": {
"taskdef": True
},
"dependencies": {
"edgelabel": "b"
},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {
"kind": "test"
},
"task": {
"task": "def"
},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {
"skip-unless-has-relevant-tests": None
},
},
},
)
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a":
Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b":
Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
simple_graph = TaskGraph(
tasks={
"a":
Task(
kind="fancy",
label="a",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b":
Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
def test_contains(self):
assert "a" in self.simple_graph
assert "c" not in self.simple_graph
def test_visit_postorder_empty(self):
"postorder visit of an empty graph is empty"
self.assertEqual(list(Graph(set(), set()).visit_postorder()), [])
def test_transitive_closure_empty(self):
"transitive closure of an empty set is an empty graph"
g = Graph({"a", "b", "c"}, {("a", "b", "L"), ("a", "c", "L")})
self.assertEqual(g.transitive_closure(set()), Graph(set(), set()))
def get_subgraph(
target_task_graph,
removed_tasks,
replaced_tasks,
label_to_taskid,
decision_task_id,
):
"""
Return the subgraph of target_task_graph consisting only of
non-optimized tasks and edges between them.
To avoid losing track of taskIds for tasks optimized away, this method
simultaneously substitutes real taskIds for task labels in the graph, and
populates each task definition's `dependencies` key with the appropriate
taskIds. Task references are resolved in the process.
"""
# check for any dependency edges from included to removed tasks
bad_edges = [(l, r, n) for l, r, n in target_task_graph.graph.edges
if l not in removed_tasks and r in removed_tasks]
if bad_edges:
probs = ", ".join(f"{l} depends on {r} as {n} but it has been removed"
for l, r, n in bad_edges)
raise Exception("Optimization error: " + probs)
# fill in label_to_taskid for anything not removed or replaced
assert replaced_tasks <= set(label_to_taskid)
for label in sorted(target_task_graph.graph.nodes - removed_tasks -
set(label_to_taskid)):
label_to_taskid[label] = slugid()
# resolve labels to taskIds and populate task['dependencies']
tasks_by_taskid = {}
named_links_dict = target_task_graph.graph.named_links_dict()
omit = removed_tasks | replaced_tasks
for label, task in target_task_graph.tasks.items():
if label in omit:
continue
task.task_id = label_to_taskid[label]
named_task_dependencies = {
name: label_to_taskid[label]
for name, label in named_links_dict.get(label, {}).items()
}
# Add remaining soft dependencies
if task.soft_dependencies:
named_task_dependencies.update({
label: label_to_taskid[label]
for label in task.soft_dependencies
if label in label_to_taskid and label not in omit
})
task.task = resolve_task_references(
task.label,
task.task,
task_id=task.task_id,
decision_task_id=decision_task_id,
dependencies=named_task_dependencies,
)
deps = task.task.setdefault("dependencies", [])
deps.extend(sorted(named_task_dependencies.values()))
tasks_by_taskid[task.task_id] = task
# resolve edges to taskIds
edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right),
name) for (left, right,
name) in target_task_graph.graph.edges)
# ..and drop edges that are no longer entirely in the task graph
# (note that this omits edges to replaced tasks, but they are still in task.dependnecies)
edges_by_taskid = {(left, right, name)
for (left, right, name) in edges_by_taskid
if left in tasks_by_taskid and right in tasks_by_taskid}
return TaskGraph(tasks_by_taskid,
Graph(set(tasks_by_taskid), edges_by_taskid))
def test_transitive_closure_disjoint(self):
"transitive closure of a disjoint set is a subset"
g = Graph({"a", "b", "c"}, set())
self.assertEqual(g.transitive_closure({"a", "c"}),
Graph({"a", "c"}, set()))
class TestGraph(unittest.TestCase):
tree = Graph(
{"a", "b", "c", "d", "e", "f", "g"},
{
("a", "b", "L"),
("a", "c", "L"),
("b", "d", "K"),
("b", "e", "K"),
("c", "f", "N"),
("c", "g", "N"),
},
)
linear = Graph(
{"1", "2", "3", "4"},
{
("1", "2", "L"),
("2", "3", "L"),
("3", "4", "L"),
},
)
diamonds = Graph(
{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"},
{
tuple(x)
for x in
"AFL ADL BDL BEL CEL CHL DFL DGL EGL EHL FIL GIL GJL HJL".split()
},
)
multi_edges = Graph(
{"1", "2", "3", "4"},
{
("2", "1", "red"),
("2", "1", "blue"),
("3", "1", "red"),
("3", "2", "blue"),
("3", "2", "green"),
("4", "3", "green"),
},
)
disjoint = Graph(
{"1", "2", "3", "4", "α", "β", "γ"},
{
("2", "1", "red"),
("3", "1", "red"),
("3", "2", "green"),
("4", "3", "green"),
("α", "β", "πράσινο"),
("β", "γ", "κόκκινο"),
("α", "γ", "μπλε"),
},
)
def test_transitive_closure_empty(self):
"transitive closure of an empty set is an empty graph"
g = Graph({"a", "b", "c"}, {("a", "b", "L"), ("a", "c", "L")})
self.assertEqual(g.transitive_closure(set()), Graph(set(), set()))
def test_transitive_closure_disjoint(self):
"transitive closure of a disjoint set is a subset"
g = Graph({"a", "b", "c"}, set())
self.assertEqual(g.transitive_closure({"a", "c"}),
Graph({"a", "c"}, set()))
def test_transitive_closure_trees(self):
"transitive closure of a tree, at two non-root nodes, is the two subtrees"
self.assertEqual(
self.tree.transitive_closure({"b", "c"}),
Graph(
{"b", "c", "d", "e", "f", "g"},
{
("b", "d", "K"),
("b", "e", "K"),
("c", "f", "N"),
("c", "g", "N"),
},
),
)
def test_transitive_closure_multi_edges(self):
"transitive closure of a tree with multiple edges between nodes keeps those edges"
self.assertEqual(
self.multi_edges.transitive_closure({"3"}),
Graph(
{"1", "2", "3"},
{
("2", "1", "red"),
("2", "1", "blue"),
("3", "1", "red"),
("3", "2", "blue"),
("3", "2", "green"),
},
),
)
def test_transitive_closure_disjoint_edges(self):
"transitive closure of a disjoint graph keeps those edges"
self.assertEqual(
self.disjoint.transitive_closure({"3", "β"}),
Graph(
{"1", "2", "3", "β", "γ"},
{
("2", "1", "red"),
("3", "1", "red"),
("3", "2", "green"),
("β", "γ", "κόκκινο"),
},
),
)
def test_transitive_closure_linear(self):
"transitive closure of a linear graph includes all nodes in the line"
self.assertEqual(self.linear.transitive_closure({"1"}), self.linear)
def test_visit_postorder_empty(self):
"postorder visit of an empty graph is empty"
self.assertEqual(list(Graph(set(), set()).visit_postorder()), [])
def assert_postorder(self, seq, all_nodes):
seen = set()
for e in seq:
for l, r, n in self.tree.edges:
if l == e:
self.assertTrue(r in seen)
seen.add(e)
self.assertEqual(seen, all_nodes)
def test_visit_postorder_tree(self):
"postorder visit of a tree satisfies invariant"
self.assert_postorder(self.tree.visit_postorder(), self.tree.nodes)
def test_visit_postorder_diamonds(self):
"postorder visit of a graph full of diamonds satisfies invariant"
self.assert_postorder(self.diamonds.visit_postorder(),
self.diamonds.nodes)
def test_visit_postorder_multi_edges(self):
"postorder visit of a graph with duplicate edges satisfies invariant"
self.assert_postorder(self.multi_edges.visit_postorder(),
self.multi_edges.nodes)
def test_visit_postorder_disjoint(self):
"postorder visit of a disjoint graph satisfies invariant"
self.assert_postorder(self.disjoint.visit_postorder(),
self.disjoint.nodes)
def assert_preorder(self, seq, all_nodes):
seen = set()
for e in seq:
for l, r, n in self.tree.edges:
if r == e:
self.assertTrue(l in seen)
seen.add(e)
self.assertEqual(seen, all_nodes)
def test_visit_preorder_tree(self):
"preorder visit of a tree satisfies invariant"
self.assert_preorder(self.tree.visit_preorder(), self.tree.nodes)
def test_visit_preorder_diamonds(self):
"preorder visit of a graph full of diamonds satisfies invariant"
self.assert_preorder(self.diamonds.visit_preorder(),
self.diamonds.nodes)
def test_visit_preorder_multi_edges(self):
"preorder visit of a graph with duplicate edges satisfies invariant"
self.assert_preorder(self.multi_edges.visit_preorder(),
self.multi_edges.nodes)
def test_visit_preorder_disjoint(self):
"preorder visit of a disjoint graph satisfies invariant"
self.assert_preorder(self.disjoint.visit_preorder(),
self.disjoint.nodes)
def test_links_dict(self):
"link dict for a graph with multiple edges is correct"
self.assertEqual(
self.multi_edges.links_dict(),
{
"2": {"1"},
"3": {"1", "2"},
"4": {"3"},
},
)
def test_named_links_dict(self):
"named link dict for a graph with multiple edges is correct"
self.assertEqual(
self.multi_edges.named_links_dict(),
{
"2": dict(red="1", blue="1"),
"3": dict(red="1", blue="2", green="2"),
"4": dict(green="3"),
},
)
def test_reverse_links_dict(self):
"reverse link dict for a graph with multiple edges is correct"
self.assertEqual(
self.multi_edges.reverse_links_dict(),
{
"1": {"2", "3"},
"2": {"3"},
"3": {"4"},
},
)