def make_task(
label,
optimization=None,
task_def=None,
optimized=None,
task_id=None,
dependencies=None,
if_dependencies=None,
):
task_def = task_def or {
"sample": "task-def",
"deadline": {
"relative-datestamp": "1 hour"
},
}
task = Task(
kind="test",
label=label,
attributes={},
task=task_def,
if_dependencies=if_dependencies or [],
)
task.optimization = optimization
task.task_id = task_id
if dependencies is not None:
task.task["dependencies"] = sorted(dependencies)
return task
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a":
Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b":
Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def test_create_tasks(self):
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
"tid-b":
Task(kind="test",
label="b",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a", "b": "tid-b"}
graph = Graph(nodes={"tid-a", "tid-b"},
edges={("tid-a", "tid-b", "edge")})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task["payload"], "hello world")
self.assertEqual(task["schedulerId"], "domain-level-4")
# make sure the dependencies exist, at least
for depid in task.get("dependencies", []):
if depid == "decisiontask":
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def make_task_graph(self):
tasks = {
"a":
Task(kind=None, label="a", attributes={}, task={}),
"b":
Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}),
"c":
Task(kind=None,
label="c",
attributes={"run_on_projects": ["try"]},
task={}),
}
graph = Graph(nodes=set("abc"), edges=set())
return TaskGraph(tasks, graph)
def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a"}
graph = Graph(nodes={"tid-a"}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError("oh noes!")
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
def default_matches(self, attributes, parameters):
method = target_tasks.get_method("default")
graph = TaskGraph(
tasks={
"a": Task(kind="build",
label="a",
attributes=attributes,
task={}),
},
graph=Graph(nodes={"a"}, edges=set()),
)
return "a" in method(graph, parameters, {})
def tg(request):
if not hasattr(request.module, "TASKS"):
pytest.fail(
"'tg' fixture used from a module that didn't define the TASKS variable"
)
tasks = request.module.TASKS
for task in tasks:
task.setdefault("task", {})
task["task"].setdefault("tags", {})
tasks = {t["label"]: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def talos_task(n, tp, bt="opt"):
return (
n,
Task(
"test",
n,
{
"talos_try_name": n,
"test_platform": tp.split("/")[0],
"build_type": bt,
},
{},
),
)
def generate_tasks(*tasks):
for i, task in enumerate(tasks):
task.setdefault("label", f"task-{i}-label")
task.setdefault("kind", "test")
task.setdefault("task", {})
task.setdefault("attributes", {})
task["attributes"].setdefault("e10s", True)
for attr in (
"optimization",
"dependencies",
"soft_dependencies",
"release_artifacts",
):
task.setdefault(attr, None)
task["task"].setdefault("label", task["label"])
yield Task.from_json(task)
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a"}
graph = Graph(nodes={"tid-a"}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task.get("dependencies"), ["decisiontask"])
def test_taskgraph_to_runnable_jobs(self):
tg, label_to_taskid = self.make_taskgraph(
{t["label"]: Task(**t)
for t in self.tasks[:]})
res = full_task_graph_to_runnable_jobs(tg.to_json())
self.assertEqual(
res,
{
"a": {
"symbol": "B"
},
"b": {
"collection": {
"opt": True
},
"groupName": "Some group",
"groupSymbol": "GS",
"symbol": "t",
"platform": "linux64",
},
},
)
def test_taskgraph_to_json(self):
tasks = {
"a":
Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b":
Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {
"attr": "a-task",
"kind": "test"
},
"task": {
"taskdef": True
},
"dependencies": {
"edgelabel": "b"
},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {
"kind": "test"
},
"task": {
"task": "def"
},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {
"skip-unless-has-relevant-tests": None
},
},
},
)
class TestTaskGraph(unittest.TestCase):
maxDiff = None
def test_taskgraph_to_json(self):
tasks = {
"a":
Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b":
Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {
"attr": "a-task",
"kind": "test"
},
"task": {
"taskdef": True
},
"dependencies": {
"edgelabel": "b"
},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {
"kind": "test"
},
"task": {
"task": "def"
},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {
"skip-unless-has-relevant-tests": None
},
},
},
)
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a":
Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b":
Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
simple_graph = TaskGraph(
tasks={
"a":
Task(
kind="fancy",
label="a",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b":
Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
def test_contains(self):
assert "a" in self.simple_graph
assert "c" not in self.simple_graph
"tasks": ["a"]
},
}
self.assertEqual(method(tg, params, {}), ["a"])
# tests for specific filters
@pytest.mark.parametrize(
"name,params,expected",
(
pytest.param(
"filter_tests_without_manifests",
{
"task": Task(kind="test", label="a", attributes={}, task={}),
"parameters": None,
},
True,
id="filter_tests_without_manifests_not_in_attributes",
),
pytest.param(
"filter_tests_without_manifests",
{
"task":
Task(
kind="test",
label="a",
attributes={"test_manifests": ["foo"]},
task={},
),
def test_make_index_tasks(make_taskgraph, graph_config):
task_def = {
"routes": [
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN",
],
"deadline":
"soon",
"metadata": {
"description": "desc",
"owner": "*****@*****.**",
"source": "https://source",
},
"extra": {
"index": {
"rank": 1540722354
},
},
}
task = Task(kind="test", label="a", attributes={}, task=task_def)
docker_task = Task(kind="docker-image",
label="docker-image-index-task",
attributes={},
task={})
taskgraph, label_to_taskid = make_taskgraph({
task.label: task,
docker_task.label: docker_task,
})
index_paths = [
r.split(".", 1)[1] for r in task_def["routes"]
if r.startswith("index.")
]
index_task = morph.make_index_task(
task,
taskgraph,
label_to_taskid,
Parameters(strict=False),
graph_config,
index_paths=index_paths,
index_rank=1540722354,
purpose="index-task",
dependencies={},
)
assert index_task.task["payload"]["command"][0] == "insert-indexes.js"
assert index_task.task["payload"]["env"]["TARGET_TASKID"] == "a-tid"
assert index_task.task["payload"]["env"]["INDEX_RANK"] == 1540722354
# check the scope summary
assert index_task.task["scopes"] == [
"index:insert-task:gecko.v2.mozilla-central.*"
]