def make_task(
label,
optimization=None,
task_def=None,
optimized=None,
task_id=None,
dependencies=None,
if_dependencies=None,
):
task_def = task_def or {
"sample": "task-def",
"deadline": {
"relative-datestamp": "1 hour"
},
}
task = Task(
kind="test",
label=label,
attributes={},
task=task_def,
if_dependencies=if_dependencies or [],
)
task.optimization = optimization
task.task_id = task_id
if dependencies is not None:
task.task["dependencies"] = sorted(dependencies)
return task
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a": Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b": Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def test_create_tasks(self):
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
"tid-b":
Task(kind="test",
label="b",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a", "b": "tid-b"}
graph = Graph(nodes={"tid-a", "tid-b"},
edges={("tid-a", "tid-b", "edge")})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task["payload"], "hello world")
self.assertEqual(task["schedulerId"], "domain-level-4")
# make sure the dependencies exist, at least
for depid in task.get("dependencies", []):
if depid == "decisiontask":
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def test_taskgraph_to_json(self):
tasks = {
'a':
Task(kind='test',
label='a',
description='Task A',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b':
Task(
kind='test',
label='b',
attributes={},
task={'task': 'def'},
optimization={'skip-unless-has-relevant-tests': None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res, {
'a': {
'kind': 'test',
'label': 'a',
'description': 'Task A',
'attributes': {
'attr': 'a-task',
'kind': 'test'
},
'task': {
'taskdef': True
},
'dependencies': {
'edgelabel': 'b'
},
'soft_dependencies': [],
'optimization': None,
},
'b': {
'kind': 'test',
'label': 'b',
'description': '',
'attributes': {
'kind': 'test'
},
'task': {
'task': 'def'
},
'dependencies': {},
'soft_dependencies': [],
'optimization': {
'skip-unless-has-relevant-tests': None
},
}
})
def test_create_tasks(self):
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
'tid-b':
Task(kind='test',
label='b',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
graph = Graph(nodes={'tid-a', 'tid-b'},
edges={('tid-a', 'tid-b', 'edge')})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task['payload'], 'hello world')
self.assertEqual(task['schedulerId'], 'domain-level-4')
# make sure the dependencies exist, at least
for depid in task.get('dependencies', []):
if depid == 'decisiontask':
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def make_task(self, label, optimization=None, task_def=None, optimized=None,
task_id=None, dependencies=None):
task_def = task_def or {'sample': 'task-def'}
task = Task(kind='test', label=label, attributes={}, task=task_def)
task.optimization = optimization
task.task_id = task_id
if dependencies is not None:
task.task['dependencies'] = sorted(dependencies)
return task
def test_make_index_tasks(self):
task_def = {
'routes': [
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN"
],
'deadline': 'soon',
'metadata': {
'description': 'desc',
'owner': '*****@*****.**',
'source': 'https://source',
},
}
task = Task(kind='test', label='a', attributes={}, task=task_def)
docker_task = Task(kind='docker-image', label='build-docker-image-index-task',
attributes={}, task={})
taskgraph, label_to_taskid = self.make_taskgraph({
task.label: task,
docker_task.label: docker_task,
})
index_task = morph.make_index_task(task, taskgraph, label_to_taskid)
self.assertEqual(index_task.task['payload']['command'][0], 'insert-indexes.js')
self.assertEqual(index_task.task['payload']['env']['TARGET_TASKID'], 'a-tid')
# check the scope summary
self.assertEqual(index_task.task['scopes'],
['index:insert-task:gecko.v2.mozilla-central.*'])
def test_taskgraph_to_json(self):
tasks = {
'a':
Task(kind='test',
label='a',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b':
Task(
kind='test',
label='b',
attributes={},
task={'task': 'def'},
optimizations=[['seta']],
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res, {
'a': {
'kind': 'test',
'label': 'a',
'attributes': {
'attr': 'a-task',
'kind': 'test'
},
'task': {
'taskdef': True
},
'dependencies': {
'edgelabel': 'b'
},
'optimizations': [],
},
'b': {
'kind': 'test',
'label': 'b',
'attributes': {
'kind': 'test'
},
'task': {
'task': 'def'
},
'dependencies': {},
'optimizations': [['seta']],
}
})
def test_taskgraph_to_json(self):
tasks = {
"a": Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b": Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {"attr": "a-task", "kind": "test"},
"task": {"taskdef": True},
"dependencies": {"edgelabel": "b"},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {"kind": "test"},
"task": {"task": "def"},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {"skip-unless-has-relevant-tests": None},
},
},
)
def make_task_graph(self):
tasks = {
'a':
Task(kind=None, label='a', attributes={}, task={}),
'b':
Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}),
'c':
Task(kind=None,
label='c',
attributes={'run_on_projects': ['try']},
task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
return TaskGraph(tasks, graph)
def make_task_graph(self):
tasks = {
"a":
Task(kind=None, label="a", attributes={}, task={}),
"b":
Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}),
"c":
Task(kind=None,
label="c",
attributes={"run_on_projects": ["try"]},
task={}),
}
graph = Graph(nodes=set("abc"), edges=set())
return TaskGraph(tasks, graph)
def test_template_env(self):
tg, label_to_taskid = self.make_taskgraph(
{t['label']: Task(**t)
for t in self.tasks[:]})
try_task_config = {
'templates': {
'env': {
'ENABLED': 1,
'FOO': 'BAZ',
}
},
}
fn = morph.apply_jsone_templates(try_task_config)
morphed = fn(tg, label_to_taskid)[0]
self.assertEqual(len(morphed.tasks), 2)
for t in morphed.tasks.values():
self.assertEqual(len(t.task['payload']['env']), 2)
self.assertEqual(t.task['payload']['env']['ENABLED'], 1)
self.assertEqual(t.task['payload']['env']['FOO'], 'BAZ')
try_task_config['templates']['env'] = {
'ENABLED': 0,
}
fn = morph.apply_jsone_templates(try_task_config)
morphed = fn(tg, label_to_taskid)[0]
self.assertEqual(len(morphed.tasks), 2)
for t in morphed.tasks.values():
self.assertEqual(len(t.task['payload']['env']), 2)
self.assertEqual(t.task['payload']['env']['ENABLED'], 0)
self.assertEqual(t.task['payload']['env']['FOO'], 'BAZ')
def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError('oh noes!')
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
def test_template_artifact(self):
tg, label_to_taskid = self.make_taskgraph(
{t['label']: Task(**t)
for t in self.tasks[:]})
try_task_config = {
'templates': {
'artifact': {
'enabled': 1
}
},
}
fn = morph.apply_jsone_templates(try_task_config)
morphed = fn(tg, label_to_taskid)[0]
self.assertEqual(len(morphed.tasks), 2)
for t in morphed.tasks.values():
if t.kind == 'build':
self.assertEqual(t.task['extra']['treeherder']['group'], 'tc')
self.assertEqual(t.task['extra']['treeherder']['symbol'], 'Ba')
self.assertEqual(t.task['payload']['env']['USE_ARTIFACT'], 1)
else:
self.assertEqual(t.task['extra']['treeherder']['group'], 'tc')
self.assertEqual(t.task['extra']['treeherder']['symbol'], 't')
self.assertNotIn('USE_ARTIFACT', t.task['payload']['env'])
def talos_task(n, tp, bt='opt'):
return (n,
Task(
'test', n, {
'talos_try_name': n,
'test_platform': tp.split('/')[0],
'build_type': bt,
}, {}))
def default_matches(self, attributes, parameters):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a':
Task(kind='build', label='a', attributes=attributes, task={}),
},
graph=Graph(nodes={'a'}, edges=set()))
return 'a' in method(graph, parameters, {})
def tg(request):
if not hasattr(request.module, 'TASKS'):
pytest.fail("'tg' fixture used from a module that didn't define the TASKS variable")
tasks = request.module.TASKS
for task in tasks:
task.setdefault('task', {})
task['task'].setdefault('tags', {})
tasks = {t['label']: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_try_tasks(self):
tasks = {
'a': Task(kind=None, label='a', attributes={}, task={}),
'b': Task(kind=None,
label='b',
attributes={'at-at': 'yep'},
task={}),
'c': Task(kind=None, label='c', attributes={}, task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
tg = TaskGraph(tasks, graph)
method = target_tasks.get_method('try_tasks')
config = os.path.join(os.getcwd(), 'try_task_config.json')
orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
try:
try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
# no try specifier
self.assertEqual(method(tg, {'message': ''}), ['b'])
# try syntax only
self.assertEqual(method(tg, {'message': 'try: me'}), ['b'])
# try task config only
with open(config, 'w') as fh:
fh.write('["c"]')
self.assertEqual(method(tg, {'message': ''}), ['c'])
with open(config, 'w') as fh:
fh.write('{"c": {}}')
self.assertEqual(method(tg, {'message': ''}), ['c'])
# both syntax and config
self.assertEqual(set(method(tg, {'message': 'try: me'})),
set(['b', 'c']))
finally:
try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
if os.path.isfile(config):
os.remove(config)
def inner(*tasks):
for i, task in enumerate(tasks):
task.setdefault('label', 'task-{}'.format(i))
task.setdefault('kind', 'test')
task.setdefault('attributes', {})
task.setdefault('task', {})
for attr in ('optimization', 'dependencies', 'soft_dependencies', 'release_artifacts'):
task.setdefault(attr, None)
task['task'].setdefault('label', task['label'])
yield Task.from_json(task)
def test_round_trip(self):
graph = TaskGraph(tasks={
'a': Task(
kind='fancy',
label='a',
description='Task A',
attributes={},
dependencies={'prereq': 'b'}, # must match edges, below
optimization={'skip-unless-has-relevant-tests': None},
task={'task': 'def'}),
'b': Task(
kind='pre',
label='b',
attributes={},
dependencies={},
optimization={'skip-unless-has-relevant-tests': None},
task={'task': 'def2'}),
}, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def default_matches(self, attributes, parameters):
method = target_tasks.get_method("default")
graph = TaskGraph(
tasks={
"a": Task(kind="build",
label="a",
attributes=attributes,
task={}),
},
graph=Graph(nodes={"a"}, edges=set()),
)
return "a" in method(graph, parameters, {})
def default_matches(self, run_on_projects, project):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a':
Task(kind='build',
label='a',
attributes={'run_on_projects': run_on_projects},
task={}),
},
graph=Graph(nodes={'a'}, edges=set()))
parameters = {'project': project}
return 'a' in method(graph, parameters)
def tg(request):
if not hasattr(request.module, "TASKS"):
pytest.fail(
"'tg' fixture used from a module that didn't define the TASKS variable"
)
tasks = request.module.TASKS
for task in tasks:
task.setdefault("task", {})
task["task"].setdefault("tags", {})
tasks = {t["label"]: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_try_tasks(self):
tasks = {
'a': Task(kind=None, label='a', attributes={}, task={}),
'b': Task(kind=None,
label='b',
attributes={'at-at': 'yep'},
task={}),
'c': Task(kind=None, label='c', attributes={}, task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
tg = TaskGraph(tasks, graph)
method = target_tasks.get_method('try_tasks')
params = {
'message': '',
'target_task_labels': [],
}
orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
try:
try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
# no try specifier
self.assertEqual(method(tg, params), ['b'])
# try syntax only
params['message'] = 'try: me'
self.assertEqual(method(tg, params), ['b'])
# try task config only
params['message'] = ''
params['target_task_labels'] = ['c']
self.assertEqual(method(tg, params), ['c'])
# both syntax and config
params['message'] = 'try: me'
self.assertEqual(set(method(tg, params)), set(['b', 'c']))
finally:
try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
os.environ['TASK_ID'] = 'decisiontask'
tasks = {
'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(GRAPH_CONFIG, taskgraph, label_to_taskid, {'level': '4'})
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def test_round_trip(self):
graph = TaskGraph(
tasks={
'a':
Task(
kind='fancy',
label='a',
attributes={},
dependencies={'prereq': 'b'}, # must match edges, below
optimizations=[['seta']],
task={'task': 'def'}),
'b':
Task(kind='pre',
label='b',
attributes={},
dependencies={},
optimizations=[['seta']],
task={'task': 'def2'}),
},
graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def make_task(self, label, optimization=None, task_def=None, optimized=None, task_id=None):
task_def = task_def or {'sample': 'task-def'}
task = Task(kind='test', label=label, attributes={}, task=task_def)
task.optimized = optimized
if optimization:
task.optimizations = [optimization]
else:
task.optimizations = []
task.task_id = task_id
return task
def generate_tasks(*tasks):
for i, task in enumerate(tasks):
task.setdefault("label", "task-{}".format(i))
task.setdefault("kind", "test")
task.setdefault("task", {})
task.setdefault("attributes", {})
task["attributes"].setdefault("e10s", True)
for attr in (
"optimization",
"dependencies",
"soft_dependencies",
"release_artifacts",
):
task.setdefault(attr, None)
task["task"].setdefault("label", task["label"])
yield Task.from_json(task)
def test_taskgraph_to_runnable_jobs(self):
tg, label_to_taskid = self.make_taskgraph({
t['label']: Task(**t) for t in self.tasks[:]
})
res = full_task_graph_to_runnable_jobs(tg.to_json())
self.assertEqual(res, {
'a': {
'symbol': 'B'
},
'b': {
'collection': {'opt': True},
'groupName': 'Some group',
'groupSymbol': 'GS',
'symbol': 't',
'platform': 'linux64'
}
})
def test_template_rebuild(self):
tg, label_to_taskid = self.make_taskgraph(
{t['label']: Task(**t)
for t in self.tasks[:]})
try_task_config = {
'tasks': ['b'],
'templates': {
'rebuild': 4,
},
}
fn = morph.apply_jsone_templates(try_task_config)
tasks = fn(tg, label_to_taskid)[0].tasks.values()
self.assertEqual(len(tasks), 2)
for t in tasks:
if t.label == 'a':
self.assertNotIn('task_duplicates', t.attributes)
elif t.label == 'b':
self.assertIn('task_duplicates', t.attributes)
self.assertEqual(t.attributes['task_duplicates'], 4)