def make_task( label, optimization=None, task_def=None, optimized=None, task_id=None, dependencies=None, if_dependencies=None, ): task_def = task_def or { "sample": "task-def", "deadline": { "relative-datestamp": "1 hour" }, } task = Task( kind="test", label=label, attributes={}, task=task_def, if_dependencies=if_dependencies or [], ) task.optimization = optimization task.task_id = task_id if dependencies is not None: task.task["dependencies"] = sorted(dependencies) return task
def test_round_trip(self): graph = TaskGraph( tasks={ "a": Task( kind="fancy", label="a", description="Task A", attributes={}, dependencies={"prereq": "b"}, # must match edges, below optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def"}, ), "b": Task( kind="pre", label="b", attributes={}, dependencies={}, optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def2"}, ), }, graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}), ) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def test_create_tasks(self): tasks = { "tid-a": Task(kind="test", label="a", attributes={}, task={"payload": "hello world"}), "tid-b": Task(kind="test", label="b", attributes={}, task={"payload": "hello world"}), } label_to_taskid = {"a": "tid-a", "b": "tid-b"} graph = Graph(nodes={"tid-a", "tid-b"}, edges={("tid-a", "tid-b", "edge")}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task["payload"], "hello world") self.assertEqual(task["schedulerId"], "domain-level-4") # make sure the dependencies exist, at least for depid in task.get("dependencies", []): if depid == "decisiontask": # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', description='Task A', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task( kind='test', label='b', attributes={}, task={'task': 'def'}, optimization={'skip-unless-has-relevant-tests': None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { 'a': { 'kind': 'test', 'label': 'a', 'description': 'Task A', 'attributes': { 'attr': 'a-task', 'kind': 'test' }, 'task': { 'taskdef': True }, 'dependencies': { 'edgelabel': 'b' }, 'soft_dependencies': [], 'optimization': None, }, 'b': { 'kind': 'test', 'label': 'b', 'description': '', 'attributes': { 'kind': 'test' }, 'task': { 'task': 'def' }, 'dependencies': {}, 'soft_dependencies': [], 'optimization': { 'skip-unless-has-relevant-tests': None }, } })
def test_create_tasks(self): tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), 'tid-b': Task(kind='test', label='b', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task['payload'], 'hello world') self.assertEqual(task['schedulerId'], 'domain-level-4') # make sure the dependencies exist, at least for depid in task.get('dependencies', []): if depid == 'decisiontask': # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def make_task(self, label, optimization=None, task_def=None, optimized=None, task_id=None, dependencies=None): task_def = task_def or {'sample': 'task-def'} task = Task(kind='test', label=label, attributes={}, task=task_def) task.optimization = optimization task.task_id = task_id if dependencies is not None: task.task['dependencies'] = sorted(dependencies) return task
def test_make_index_tasks(self): task_def = { 'routes': [ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN" ], 'deadline': 'soon', 'metadata': { 'description': 'desc', 'owner': '*****@*****.**', 'source': 'https://source', }, } task = Task(kind='test', label='a', attributes={}, task=task_def) docker_task = Task(kind='docker-image', label='build-docker-image-index-task', attributes={}, task={}) taskgraph, label_to_taskid = self.make_taskgraph({ task.label: task, docker_task.label: docker_task, }) index_task = morph.make_index_task(task, taskgraph, label_to_taskid) self.assertEqual(index_task.task['payload']['command'][0], 'insert-indexes.js') self.assertEqual(index_task.task['payload']['env']['TARGET_TASKID'], 'a-tid') # check the scope summary self.assertEqual(index_task.task['scopes'], ['index:insert-task:gecko.v2.mozilla-central.*'])
def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task( kind='test', label='b', attributes={}, task={'task': 'def'}, optimizations=[['seta']], # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { 'a': { 'kind': 'test', 'label': 'a', 'attributes': { 'attr': 'a-task', 'kind': 'test' }, 'task': { 'taskdef': True }, 'dependencies': { 'edgelabel': 'b' }, 'optimizations': [], }, 'b': { 'kind': 'test', 'label': 'b', 'attributes': { 'kind': 'test' }, 'task': { 'task': 'def' }, 'dependencies': {}, 'optimizations': [['seta']], } })
def test_taskgraph_to_json(self): tasks = { "a": Task( kind="test", label="a", description="Task A", attributes={"attr": "a-task"}, task={"taskdef": True}, ), "b": Task( kind="test", label="b", attributes={}, task={"task": "def"}, optimization={"skip-unless-has-relevant-tests": None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={"first": "a"}, ), } graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { "a": { "kind": "test", "label": "a", "description": "Task A", "attributes": {"attr": "a-task", "kind": "test"}, "task": {"taskdef": True}, "dependencies": {"edgelabel": "b"}, "soft_dependencies": [], "if_dependencies": [], "optimization": None, }, "b": { "kind": "test", "label": "b", "description": "", "attributes": {"kind": "test"}, "task": {"task": "def"}, "dependencies": {}, "soft_dependencies": [], "if_dependencies": [], "optimization": {"skip-unless-has-relevant-tests": None}, }, }, )
def make_task_graph(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={'run_on_projects': ['try']}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) return TaskGraph(tasks, graph)
def make_task_graph(self): tasks = { "a": Task(kind=None, label="a", attributes={}, task={}), "b": Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}), "c": Task(kind=None, label="c", attributes={"run_on_projects": ["try"]}, task={}), } graph = Graph(nodes=set("abc"), edges=set()) return TaskGraph(tasks, graph)
def test_template_env(self): tg, label_to_taskid = self.make_taskgraph( {t['label']: Task(**t) for t in self.tasks[:]}) try_task_config = { 'templates': { 'env': { 'ENABLED': 1, 'FOO': 'BAZ', } }, } fn = morph.apply_jsone_templates(try_task_config) morphed = fn(tg, label_to_taskid)[0] self.assertEqual(len(morphed.tasks), 2) for t in morphed.tasks.values(): self.assertEqual(len(t.task['payload']['env']), 2) self.assertEqual(t.task['payload']['env']['ENABLED'], 1) self.assertEqual(t.task['payload']['env']['FOO'], 'BAZ') try_task_config['templates']['env'] = { 'ENABLED': 0, } fn = morph.apply_jsone_templates(try_task_config) morphed = fn(tg, label_to_taskid)[0] self.assertEqual(len(morphed.tasks), 2) for t in morphed.tasks.values(): self.assertEqual(len(t.task['payload']['env']), 2) self.assertEqual(t.task['payload']['env']['ENABLED'], 0) self.assertEqual(t.task['payload']['env']['FOO'], 'BAZ')
def test_create_tasks_fails_if_create_fails(self, create_task): "creat_tasks fails if a single create_task call fails" tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) def fail(*args): print("UHOH") raise RuntimeError('oh noes!') create_task.side_effect = fail with self.assertRaises(RuntimeError): create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", )
def test_template_artifact(self): tg, label_to_taskid = self.make_taskgraph( {t['label']: Task(**t) for t in self.tasks[:]}) try_task_config = { 'templates': { 'artifact': { 'enabled': 1 } }, } fn = morph.apply_jsone_templates(try_task_config) morphed = fn(tg, label_to_taskid)[0] self.assertEqual(len(morphed.tasks), 2) for t in morphed.tasks.values(): if t.kind == 'build': self.assertEqual(t.task['extra']['treeherder']['group'], 'tc') self.assertEqual(t.task['extra']['treeherder']['symbol'], 'Ba') self.assertEqual(t.task['payload']['env']['USE_ARTIFACT'], 1) else: self.assertEqual(t.task['extra']['treeherder']['group'], 'tc') self.assertEqual(t.task['extra']['treeherder']['symbol'], 't') self.assertNotIn('USE_ARTIFACT', t.task['payload']['env'])
def talos_task(n, tp, bt='opt'): return (n, Task( 'test', n, { 'talos_try_name': n, 'test_platform': tp.split('/')[0], 'build_type': bt, }, {}))
def default_matches(self, attributes, parameters): method = target_tasks.get_method('default') graph = TaskGraph(tasks={ 'a': Task(kind='build', label='a', attributes=attributes, task={}), }, graph=Graph(nodes={'a'}, edges=set())) return 'a' in method(graph, parameters, {})
def tg(request): if not hasattr(request.module, 'TASKS'): pytest.fail("'tg' fixture used from a module that didn't define the TASKS variable") tasks = request.module.TASKS for task in tasks: task.setdefault('task', {}) task['task'].setdefault('tags', {}) tasks = {t['label']: Task(**t) for t in tasks} return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_try_tasks(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) tg = TaskGraph(tasks, graph) method = target_tasks.get_method('try_tasks') config = os.path.join(os.getcwd(), 'try_task_config.json') orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax try: try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax # no try specifier self.assertEqual(method(tg, {'message': ''}), ['b']) # try syntax only self.assertEqual(method(tg, {'message': 'try: me'}), ['b']) # try task config only with open(config, 'w') as fh: fh.write('["c"]') self.assertEqual(method(tg, {'message': ''}), ['c']) with open(config, 'w') as fh: fh.write('{"c": {}}') self.assertEqual(method(tg, {'message': ''}), ['c']) # both syntax and config self.assertEqual(set(method(tg, {'message': 'try: me'})), set(['b', 'c'])) finally: try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax if os.path.isfile(config): os.remove(config)
def inner(*tasks): for i, task in enumerate(tasks): task.setdefault('label', 'task-{}'.format(i)) task.setdefault('kind', 'test') task.setdefault('attributes', {}) task.setdefault('task', {}) for attr in ('optimization', 'dependencies', 'soft_dependencies', 'release_artifacts'): task.setdefault(attr, None) task['task'].setdefault('label', task['label']) yield Task.from_json(task)
def test_round_trip(self): graph = TaskGraph(tasks={ 'a': Task( kind='fancy', label='a', description='Task A', attributes={}, dependencies={'prereq': 'b'}, # must match edges, below optimization={'skip-unless-has-relevant-tests': None}, task={'task': 'def'}), 'b': Task( kind='pre', label='b', attributes={}, dependencies={}, optimization={'skip-unless-has-relevant-tests': None}, task={'task': 'def2'}), }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def default_matches(self, attributes, parameters): method = target_tasks.get_method("default") graph = TaskGraph( tasks={ "a": Task(kind="build", label="a", attributes=attributes, task={}), }, graph=Graph(nodes={"a"}, edges=set()), ) return "a" in method(graph, parameters, {})
def default_matches(self, run_on_projects, project): method = target_tasks.get_method('default') graph = TaskGraph(tasks={ 'a': Task(kind='build', label='a', attributes={'run_on_projects': run_on_projects}, task={}), }, graph=Graph(nodes={'a'}, edges=set())) parameters = {'project': project} return 'a' in method(graph, parameters)
def tg(request): if not hasattr(request.module, "TASKS"): pytest.fail( "'tg' fixture used from a module that didn't define the TASKS variable" ) tasks = request.module.TASKS for task in tasks: task.setdefault("task", {}) task["task"].setdefault("tags", {}) tasks = {t["label"]: Task(**t) for t in tasks} return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_try_tasks(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) tg = TaskGraph(tasks, graph) method = target_tasks.get_method('try_tasks') params = { 'message': '', 'target_task_labels': [], } orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax try: try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax # no try specifier self.assertEqual(method(tg, params), ['b']) # try syntax only params['message'] = 'try: me' self.assertEqual(method(tg, params), ['b']) # try task config only params['message'] = '' params['target_task_labels'] = ['c'] self.assertEqual(method(tg, params), ['c']) # both syntax and config params['message'] = 'try: me' self.assertEqual(set(method(tg, params)), set(['b', 'c'])) finally: try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" os.environ['TASK_ID'] = 'decisiontask' tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks(GRAPH_CONFIG, taskgraph, label_to_taskid, {'level': '4'}) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def test_round_trip(self): graph = TaskGraph( tasks={ 'a': Task( kind='fancy', label='a', attributes={}, dependencies={'prereq': 'b'}, # must match edges, below optimizations=[['seta']], task={'task': 'def'}), 'b': Task(kind='pre', label='b', attributes={}, dependencies={}, optimizations=[['seta']], task={'task': 'def2'}), }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def make_task(self, label, optimization=None, task_def=None, optimized=None, task_id=None): task_def = task_def or {'sample': 'task-def'} task = Task(kind='test', label=label, attributes={}, task=task_def) task.optimized = optimized if optimization: task.optimizations = [optimization] else: task.optimizations = [] task.task_id = task_id return task
def generate_tasks(*tasks): for i, task in enumerate(tasks): task.setdefault("label", "task-{}".format(i)) task.setdefault("kind", "test") task.setdefault("task", {}) task.setdefault("attributes", {}) task["attributes"].setdefault("e10s", True) for attr in ( "optimization", "dependencies", "soft_dependencies", "release_artifacts", ): task.setdefault(attr, None) task["task"].setdefault("label", task["label"]) yield Task.from_json(task)
def test_taskgraph_to_runnable_jobs(self): tg, label_to_taskid = self.make_taskgraph({ t['label']: Task(**t) for t in self.tasks[:] }) res = full_task_graph_to_runnable_jobs(tg.to_json()) self.assertEqual(res, { 'a': { 'symbol': 'B' }, 'b': { 'collection': {'opt': True}, 'groupName': 'Some group', 'groupSymbol': 'GS', 'symbol': 't', 'platform': 'linux64' } })
def test_template_rebuild(self): tg, label_to_taskid = self.make_taskgraph( {t['label']: Task(**t) for t in self.tasks[:]}) try_task_config = { 'tasks': ['b'], 'templates': { 'rebuild': 4, }, } fn = morph.apply_jsone_templates(try_task_config) tasks = fn(tg, label_to_taskid)[0].tasks.values() self.assertEqual(len(tasks), 2) for t in tasks: if t.label == 'a': self.assertNotIn('task_duplicates', t.attributes) elif t.label == 'b': self.assertIn('task_duplicates', t.attributes) self.assertEqual(t.attributes['task_duplicates'], 4)