def generate_tasks(params=None, full=False, disable_target_task_filter=False):
cache_dir = os.path.join(get_state_dir(srcdir=True), "cache", "taskgraph")
attr = "full_task_set" if full else "target_task_set"
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, "r") as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace("_", " ")))
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(build.topsrcdir, "taskcluster", "ci")
target_tasks_method = ("try_select_tasks" if not disable_target_task_filter
else "try_select_tasks_uncommon")
params = parameters_loader(
params,
strict=False,
overrides={
"try_mode": "try_select",
"target_tasks_method": target_tasks_method,
},
)
# Cache both full_task_set and target_task_set regardless of whether or not
# --full was requested. Caching is cheap and can potentially save a lot of
# time.
generator = TaskGraphGenerator(root_dir=root, parameters=params)
def generate(attr):
try:
tg = getattr(generator, attr)
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
# write cache
with open(os.path.join(cache_dir, attr), "w") as fh:
json.dump(tg.to_json(), fh)
return tg
tg_full = generate("full_task_set")
tg_target = generate("target_task_set")
# discard results from these, we only need cache.
if full:
generate("full_task_graph")
generate("target_task_graph")
os.chdir(cwd)
if full:
return tg_full
return tg_target
def generate_tasks(params=None, full=False):
params = params or "project=mozilla-central"
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace('_', ' ')))
params = load_parameters_file(params)
params.check()
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
try:
params = load_parameters_file(params, strict=False)
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
os.chdir(cwd)
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def generate_tasks(params=None, full=False):
# TODO: Remove after January 1st, 2020.
# Try to delete the old taskgraph cache directories.
root = build.topsrcdir
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
old_cache_dirs = [
os.path.join(get_state_dir(), 'cache', 'taskgraph'),
os.path.join(get_state_dir(), 'cache', root_hash, 'taskgraph'),
]
for cache_dir in old_cache_dirs:
if os.path.isdir(cache_dir):
shutil.rmtree(cache_dir)
cache_dir = os.path.join(get_state_dir(srcdir=True), 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(root)
root = os.path.join(root, 'taskcluster', 'ci')
params = parameters_loader(params, strict=False, overrides={'try_mode': 'try_select'})
# Cache both full_task_set and target_task_set regardless of whether or not
# --full was requested. Caching is cheap and can potentially save a lot of
# time.
generator = TaskGraphGenerator(root_dir=root, parameters=params)
def generate(attr):
try:
tg = getattr(generator, attr)
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
# write cache
with open(os.path.join(cache_dir, attr), 'w') as fh:
json.dump(tg.to_json(), fh)
return tg
tg_full = generate('full_task_set')
tg_target = generate('target_task_set')
os.chdir(cwd)
if full:
return tg_full
return tg_target
def generate_tasks(params=None, full=False):
cache_dir = os.path.join(get_state_dir(srcdir=True), 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
params = parameters_loader(params, strict=False, overrides={'try_mode': 'try_select'})
# Cache both full_task_set and target_task_set regardless of whether or not
# --full was requested. Caching is cheap and can potentially save a lot of
# time.
generator = TaskGraphGenerator(root_dir=root, parameters=params)
def generate(attr):
try:
tg = getattr(generator, attr)
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
# write cache
with open(os.path.join(cache_dir, attr), 'w') as fh:
json.dump(tg.to_json(), fh)
return tg
tg_full = generate('full_task_set')
tg_target = generate('target_task_set')
# discard results from these, we only need cache.
if full:
generate('full_task_graph')
generate('target_task_graph')
os.chdir(cwd)
if full:
return tg_full
return tg_target
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
# Try to delete the old taskgraph cache directory.
old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
if os.path.isdir(old_cache_dir):
shutil.rmtree(old_cache_dir)
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash,
'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace('_', ' ')))
try:
params = load_parameters_file(params,
strict=False,
overrides={'try_mode': 'try_select'})
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]
if not full:
labels = filter(filter_target_task, labels)
os.chdir(cwd)
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def generate_tasks(params, full, root):
params = params or "project=mozilla-central"
# Try to delete the old taskgraph cache directory.
old_cache_dir = os.path.join(get_state_dir(), 'cache', 'taskgraph')
if os.path.isdir(old_cache_dir):
shutil.rmtree(old_cache_dir)
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
cache_dir = os.path.join(get_state_dir(), 'cache', root_hash, 'taskgraph')
# Cleanup old cache files
for path in glob.glob(os.path.join(cache_dir, '*_set')):
os.remove(path)
attr = 'full_task_graph' if full else 'target_task_graph'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace('_', ' ')))
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(root, 'taskcluster', 'ci')
params = parameters_loader(params,
strict=False,
overrides={'try_mode': 'try_select'})
try:
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params),
attr)
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
os.chdir(cwd)
with open(cache, 'w') as fh:
json.dump(tg.to_json(), fh)
return tg
def inner(parameters=None, overrides=None):
params = parameters_loader(parameters,
strict=False,
overrides=overrides)
tgg = TaskGraphGenerator(None, params)
# Mock out certain requests as they may depend on a revision that does
# not exist on hg.mozilla.org.
mock_requests = {}
# bugbug /push/schedules
url = BUGBUG_BASE_URL + "/push/{project}/{head_rev}/schedules".format(
**tgg.parameters)
mock_requests[url] = "bugbug-push-schedules.json"
# files changed
url = "{head_repository}/json-automationrelevance/{head_rev}".format(
**tgg.parameters)
mock_requests[url] = "automationrelevance.json"
url = PUSHLOG_PUSHES_TMPL.format(
repository=tgg.parameters["head_repository"],
push_id_start=int(tgg.parameters["pushlog_id"]) - 2,
push_id_end=int(tgg.parameters["pushlog_id"]) - 1,
)
mock_requests[url] = "pushes.json"
for url, filename in mock_requests.items():
with open(os.path.join(datadir, filename)) as fh:
responses.add(
responses.GET,
url,
json=json.load(fh),
status=200,
)
# Still allow other real requests.
responses.add_passthru("https://hg.mozilla.org")
responses.add_passthru("https://firefox-ci-tc.services.mozilla.com")
return tgg
def generate_target(params='project=mozilla-central'):
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
cache = os.path.join(cache_dir, 'target_task_set')
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating target tasks")
params = load_parameters_file(params)
params.check()
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = TaskGraphGenerator(root_dir=root, parameters=params).target_task_set
labels = [label for label in tg.graph.visit_postorder()]
with open(cache, 'w') as fh:
fh.write('\n'.join(labels))
return labels
def get_taskgraph_generator(root, parameters):
"""Helper function to make testing a little easier."""
from taskgraph.generator import TaskGraphGenerator
return TaskGraphGenerator(root_dir=root, parameters=parameters)