def generate_tasks(params=None, full=False): params = params or "project=mozilla-central" cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph') attr = 'full_task_set' if full else 'target_task_set' cache = os.path.join(cache_dir, attr) invalidate(cache) if os.path.isfile(cache): with open(cache, 'r') as fh: return fh.read().splitlines() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format( attr.replace('_', ' '))) params = load_parameters_file(params) params.check() root = os.path.join(build.topsrcdir, 'taskcluster', 'ci') tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr) labels = [label for label in tg.graph.visit_postorder()] with open(cache, 'w') as fh: fh.write('\n'.join(labels)) return labels
def test_load_parameters_override(self): """ When ``load_parameters_file`` is passed overrides, they are included in the generated parameters. """ self.assertEqual(load_parameters_file('', overrides={'some': 'data'}), {'some': 'data'})
def generate_tasks(params, full, root): params = params or "project=mozilla-central" cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph') attr = 'full_task_set' if full else 'target_task_set' cache = os.path.join(cache_dir, attr) invalidate(cache, root) if os.path.isfile(cache): with open(cache, 'r') as fh: return fh.read().splitlines() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format(attr.replace('_', ' '))) try: params = load_parameters_file(params, strict=False) params.check() except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) cwd = os.getcwd() os.chdir(build.topsrcdir) root = os.path.join(root, 'taskcluster', 'ci') tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr) labels = [label for label in tg.graph.visit_postorder()] os.chdir(cwd) with open(cache, 'w') as fh: fh.write('\n'.join(labels)) return labels
def test_load_parameters_override(self): """ When ``load_parameters_file`` is passed overrides, they are included in the generated parameters. """ self.assertEqual(load_parameters_file("", overrides={"some": "data"}), {"some": "data"})
def test_load_parameters_override_file(self): """ When ``load_parameters_file`` is passed overrides, they overwrite data loaded from a file. """ with MockedOpen({"params.json": '{"some": "data"}'}): self.assertEqual( load_parameters_file('params.json', overrides={'some': 'other'}), {'some': 'other'})
def generate_tasks(params, full, root): params = params or "project=mozilla-central" # Try to delete the old taskgraph cache directory. old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph') if os.path.isdir(old_cache_dir): shutil.rmtree(old_cache_dir) root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest() cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash, 'taskgraph') attr = 'full_task_set' if full else 'target_task_set' cache = os.path.join(cache_dir, attr) invalidate(cache, root) if os.path.isfile(cache): with open(cache, 'r') as fh: return fh.read().splitlines() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format( attr.replace('_', ' '))) try: params = load_parameters_file(params, strict=False, overrides={'try_mode': 'try_select'}) params.check() except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) taskgraph.fast = True cwd = os.getcwd() os.chdir(build.topsrcdir) root = os.path.join(root, 'taskcluster', 'ci') tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr) labels = [label for label in tg.graph.visit_postorder()] if not full: labels = filter(filter_target_task, labels) os.chdir(cwd) with open(cache, 'w') as fh: fh.write('\n'.join(labels)) return labels
def generate_tasks(params, full, root): params = params or "project=mozilla-central" # Try to delete the old taskgraph cache directory. old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph') if os.path.isdir(old_cache_dir): shutil.rmtree(old_cache_dir) root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest() cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash, 'taskgraph') # Cleanup old cache files for path in glob.glob(os.path.join(cache_dir, '*_set')): os.remove(path) attr = 'full_task_graph' if full else 'target_task_graph' cache = os.path.join(cache_dir, attr) invalidate(cache, root) if os.path.isfile(cache): with open(cache, 'r') as fh: return TaskGraph.from_json(json.load(fh))[1] if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format(attr.replace('_', ' '))) try: params = load_parameters_file(params, strict=False, overrides={'try_mode': 'try_select'}) params.check() except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) taskgraph.fast = True cwd = os.getcwd() os.chdir(build.topsrcdir) root = os.path.join(root, 'taskcluster', 'ci') tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr) os.chdir(cwd) with open(cache, 'w') as fh: json.dump(tg.to_json(), fh) return tg
def generate_target(params='project=mozilla-central'): cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph') cache = os.path.join(cache_dir, 'target_task_set') invalidate(cache) if os.path.isfile(cache): with open(cache, 'r') as fh: return fh.read().splitlines() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating target tasks") params = load_parameters_file(params) params.check() root = os.path.join(build.topsrcdir, 'taskcluster', 'ci') tg = TaskGraphGenerator(root_dir=root, parameters=params).target_task_set labels = [label for label in tg.graph.visit_postorder()] with open(cache, 'w') as fh: fh.write('\n'.join(labels)) return labels
def generate_tasks(params, full, root): params = params or "project=mozilla-central" cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph') attr = 'full_task_set' if full else 'target_task_set' cache = os.path.join(cache_dir, attr) invalidate(cache, root) if os.path.isfile(cache): with open(cache, 'r') as fh: return fh.read().splitlines() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format(attr.replace('_', ' '))) try: params = load_parameters_file(params, strict=False) params.check() except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) taskgraph.fast = True cwd = os.getcwd() os.chdir(build.topsrcdir) root = os.path.join(root, 'taskcluster', 'ci') tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr) labels = [label for label in tg.graph.visit_postorder()] os.chdir(cwd) with open(cache, 'w') as fh: fh.write('\n'.join(labels)) return labels
def test_load_parameters_file_json(self): with MockedOpen({"params.json": '{"some": "data"}'}): self.assertEqual(load_parameters_file('params.json'), {'some': 'data'})
def test_load_parameters_file_yaml(self): with MockedOpen({"params.yml": "some: data\n"}): self.assertEqual(load_parameters_file('params.yml'), {'some': 'data'})
def test_load_parameters_file_yaml(self): with MockedOpen({"params.yml": "some: data\n"}): self.assertEqual(load_parameters_file("params.yml"), {"some": "data"})
def test_load_parameters_file_json(self): with MockedOpen({"params.json": '{"some": "data"}'}): self.assertEqual( load_parameters_file('params.json'), {'some': 'data'})
def test_load_parameters_file_yaml(self): with MockedOpen({"params.yml": "some: data\n"}): self.assertEqual( load_parameters_file('params.yml'), {'some': 'data'})