def generate_tasks(params=None, full=False):
params = params or "project=mozilla-central"
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace('_', ' ')))
params = load_parameters_file(params)
params.check()
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def test_load_parameters_override(self):
"""
When ``load_parameters_file`` is passed overrides, they are included in
the generated parameters.
"""
self.assertEqual(load_parameters_file('', overrides={'some': 'data'}),
{'some': 'data'})
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
try:
params = load_parameters_file(params, strict=False)
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
os.chdir(cwd)
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def test_load_parameters_override(self):
"""
When ``load_parameters_file`` is passed overrides, they are included in
the generated parameters.
"""
self.assertEqual(load_parameters_file("", overrides={"some": "data"}),
{"some": "data"})
def test_load_parameters_override_file(self):
"""
When ``load_parameters_file`` is passed overrides, they overwrite data
loaded from a file.
"""
with MockedOpen({"params.json": '{"some": "data"}'}):
self.assertEqual(
load_parameters_file('params.json', overrides={'some': 'other'}),
{'some': 'other'})
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
# Try to delete the old taskgraph cache directory.
old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
if os.path.isdir(old_cache_dir):
shutil.rmtree(old_cache_dir)
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash,
'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace('_', ' ')))
try:
params = load_parameters_file(params,
strict=False,
overrides={'try_mode': 'try_select'})
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
if not full:
labels = filter(filter_target_task, labels)
os.chdir(cwd)
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
# Try to delete the old taskgraph cache directory.
old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
if os.path.isdir(old_cache_dir):
shutil.rmtree(old_cache_dir)
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash, 'taskgraph')
# Cleanup old cache files
for path in glob.glob(os.path.join(cache_dir, '*_set')):
os.remove(path)
attr = 'full_task_graph' if full else 'target_task_graph'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
try:
params = load_parameters_file(params, strict=False, overrides={'try_mode': 'try_select'})
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
os.chdir(cwd)
with open(cache, 'w') as fh:
json.dump(tg.to_json(), fh)
return tg
def generate_target(params='project=mozilla-central'):
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
cache = os.path.join(cache_dir, 'target_task_set')
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating target tasks")
params = load_parameters_file(params)
params.check()
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = TaskGraphGenerator(root_dir=root, parameters=params).target_task_set
labels = [label for label in tg.graph.visit_postorder()]
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
try:
params = load_parameters_file(params, strict=False)
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
os.chdir(cwd)
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def test_load_parameters_file_json(self):
with MockedOpen({"params.json": '{"some": "data"}'}):
self.assertEqual(load_parameters_file('params.json'),
{'some': 'data'})
def test_load_parameters_file_yaml(self):
with MockedOpen({"params.yml": "some: data\n"}):
self.assertEqual(load_parameters_file('params.yml'),
{'some': 'data'})
def test_load_parameters_file_yaml(self):
with MockedOpen({"params.yml": "some: data\n"}):
self.assertEqual(load_parameters_file("params.yml"),
{"some": "data"})
def test_load_parameters_file_json(self):
with MockedOpen({"params.json": '{"some": "data"}'}):
self.assertEqual(
load_parameters_file('params.json'),
{'some': 'data'})
def test_load_parameters_file_yaml(self):
with MockedOpen({"params.yml": "some: data\n"}):
self.assertEqual(
load_parameters_file('params.yml'),
{'some': 'data'})
