def create_task_payload(build, base_context):
build_type = os.path.splitext(os.path.basename(build))[0]
build_context = defaultValues_build_context()
with open(build) as src:
build_context['build'].update(
yaml.load(src, Loader=yaml.FullLoader)['build'])
# Be able to use what has been defined in base_context
# e.g., the {${event.head.branch}}
build_context = jsone.render(build_context, base_context)
template_context = {
'taskcluster': {
'taskId': as_slugid(build_type)
},
'build_type': build_type
}
with open(os.path.join(TASKS_ROOT,
build_context['build']['template_file'])) as src:
template = yaml.load(src, Loader=yaml.FullLoader)
contextes = merge_dicts({}, base_context, template_context, build_context)
for one_context in glob(os.path.join(TASKS_ROOT, '*.cyml')):
with open(one_context) as src:
contextes = merge_dicts(contextes,
yaml.load(src, Loader=yaml.FullLoader))
return jsone.render(template, contextes)
def test_jsone_validates(tmp_path, hook_file, payload):
tmp_hook_file = tmp_path / "hook.json"
shutil.copyfile(hook_file, tmp_hook_file)
set_hook(tmp_hook_file, version)
with open(tmp_hook_file, "r") as f:
hook_content = json.load(f)
jsonschema.validate(instance=payload, schema=hook_content["triggerSchema"])
jsone.render(hook_content, context={"payload": payload})
def test_jsone_validates(tmp_path):
payload = {}
hook_file = os.path.realpath("taskcluster-hook.json")
assert os.path.exists(hook_file)
content = open(hook_file).read()
content = content.replace("CHANNEL", "dev")
content = content.replace("VERSION", version)
hook_content = json.loads(content)
jsonschema.validate(instance=payload, schema=hook_content["triggerSchema"])
jsone.render(hook_content, context={"payload": payload})
def test_cron(self):
context = {
"tasks_for": "cron",
"repository": {
"url": "https://hg.mozilla.org/mozilla-central",
"project": "mozilla-central",
"level": 3,
},
"push": {
"revision": "e8aebe488b2f2e567940577de25013d00e818f7c",
"pushlog_id": -1,
"pushdate": 0,
"owner": "cron",
},
"cron": {
"task_id": "<cron task id>",
"job_name": "test",
"job_symbol": "T",
"quoted_args": "abc def",
},
"now": current_json_time(),
"ownTaskId": slugid.nice().encode("ascii"),
}
rendered = jsone.render(self.taskcluster_yml, context)
pprint.pprint(rendered)
self.assertEqual(rendered["tasks"][0]["metadata"]["name"],
"Decision Task for cron job test")
def test_action(self):
context = {
"tasks_for": "action",
"repository": {
"url": "https://hg.mozilla.org/mozilla-central",
"project": "mozilla-central",
"level": 3,
},
"push": {
"revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20",
"owner": "*****@*****.**",
"pushlog_id": 1556565286,
"pushdate": 112957,
},
"action": {
"name": "test-action",
"title": "Test Action",
"description": "Just testing",
"taskGroupId": slugid.nice().encode("ascii"),
"symbol": "t",
"repo_scope": "assume:repo:hg.mozilla.org/try:action:generic",
"cb_name": "test_action",
},
"input": {},
"parameters": {},
"now": current_json_time(),
"taskId": slugid.nice().encode("ascii"),
"ownTaskId": slugid.nice().encode("ascii"),
"clientId": "testing/testing/testing",
}
rendered = jsone.render(self.taskcluster_yml, context)
pprint.pprint(rendered)
self.assertEqual(rendered["tasks"][0]["metadata"]["name"],
"Action: Test Action")
def py(when, template, context):
when = datetime.datetime.utcfromtimestamp(when)
with freeze_time(when):
try:
return jsone.render(template, context)
except jsone.JSONTemplateError:
return Exception
def load_config(context):
path_to_worker_template = os.path.join(os.path.dirname(__file__), "..",
"docker.d", "worker.yml")
with open(path_to_worker_template, "r") as file:
worker_template = yaml.safe_load(file)
return jsone.render(worker_template, context)
def test_jsone_validates(pipeline_file, task_schema, payload_schema):
responses.add_passthru("https://community-tc.services.mozilla.com/")
with open(pipeline_file, "r") as f:
yaml_content = yaml.safe_load(f.read())
result = jsone.render(yaml_content, context={"version": "42.0"})
tasks = result["tasks"]
all_ids = [task["ID"] for task in tasks]
# Make sure there are no duplicate IDs.
assert len(all_ids) == len(set(all_ids))
# Make sure all dependencies are present.
for task in tasks:
assert "dependencies" not in task or all(
dependency in all_ids for dependency in task["dependencies"])
for task in tasks:
if "ID" in task:
del task["ID"]
if "dependencies" in task:
del task["dependencies"]
jsonschema.validate(instance=task, schema=task_schema)
jsonschema.validate(instance=task["payload"], schema=payload_schema)
def __call__(self, taskgraph, label_to_taskid):
if not self.templates:
return taskgraph, label_to_taskid
for task in taskgraph.tasks.itervalues():
for template in sorted(self.templates):
context = {
'task': task.task,
'taskGroup': None,
'taskId': task.task_id,
'kind': task.kind,
'input': self.templates[template],
# The following context differs from action tasks
'attributes': task.attributes,
'label': task.label,
'target_tasks': self.target_tasks,
}
template_path = os.path.join(self.template_dir, template + '.yml')
with open(template_path) as f:
template = yaml.load(f)
result = jsone.render(template, context) or {}
for attr in ('task', 'attributes'):
if attr in result:
setattr(task, attr, result[attr])
return taskgraph, label_to_taskid
def _submit(self,
action=None,
decision_task_id=None,
task_id=None,
input=None,
static_action_variables=None) -> str:
context = {
"taskGroupId": decision_task_id,
"taskId": task_id or None,
"input": input,
}
context.update(static_action_variables)
action_kind = action["kind"]
if action_kind == "hook":
hook_payload = jsone.render(action["hookPayload"], context)
hook_id, hook_group_id = action["hookId"], action["hookGroupId"]
decision_task = self.queue.task(decision_task_id)
expansion = self.auth.expandScopes({"scopes": decision_task["scopes"]})
expression = f"in-tree:hook-action:{hook_group_id}/{hook_id}"
if not satisfiesExpression(expansion["scopes"], expression):
raise RuntimeError(f"Action is misconfigured: decision task's scopes do not satisfy {expression}")
result = self.hooks.triggerHook(hook_group_id, hook_id, hook_payload)
return result["status"]["taskId"]
raise NotImplementedError(f"Unable to submit actions with '{action_kind}' kind.")
def trigger_action(action_name, decision_task_id, task_id=None, input={}):
if not decision_task_id:
raise ValueError(
"No decision task. We can't find the actions artifact.")
actions_json = get_artifact(decision_task_id, "public/actions.json")
if actions_json["version"] != 1:
raise RuntimeError("Wrong version of actions.json, unable to continue")
# These values substitute $eval in the template
context = {
"input": input,
"taskId": task_id,
"taskGroupId": decision_task_id,
}
# https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#variables
context.update(actions_json["variables"])
action = _extract_applicable_action(actions_json, action_name,
decision_task_id, task_id)
kind = action["kind"]
if kind == "hook":
hook_payload = jsone.render(action["hookPayload"], context)
trigger_hook(action["hookGroupId"], action["hookId"], hook_payload)
else:
raise NotImplementedError(
"Unable to submit actions with {} kind.".format(kind))
def retrigger(self, push, times=3):
"""This function implements ability to perform retriggers on tasks"""
if self._should_retrigger() == "false":
logger.info(
"Not retriggering task '{}', task should not be retriggered".
format(self.tags.get("label")))
return None
decision_task = push.decision_task
retrigger_action = self._get_action(decision_task, "retrigger")
hook_payload = jsone.render(
retrigger_action["hookPayload"],
context={
"taskId": self.id,
"taskGroupId": decision_task.id,
"input": {
"times": times
},
},
)
logger.info("Retriggering task '{}'".format(self.tags.get("label",
"")))
return self._trigger_action(retrigger_action, hook_payload)
def run_query(name, config, **context):
"""Loads and runs the specified query, yielding the result.
Given name of a query, this method will first read the query
from a .query file corresponding to the name.
After queries are loaded, each query to be run is inspected
and overridden if the provided context has values for limit.
The actual call to the ActiveData endpoint is encapsulated
inside the query_activedata method.
:param str name: name of the query file to be loaded.
:param Configuration config: config object.
:param dict context: dictionary of ActiveData configs.
:yields str: json-formatted string.
"""
for query in load_query(name):
# If limit is in the context, override the queries' value. We do this
# to keep the results down to a sane level when testing queries.
if 'limit' in context:
query['limit'] = context['limit']
if 'format' in context:
query['format'] = context['format']
if config.debug:
query['meta'] = {"save": True}
query = jsone.render(query, context)
query_str = json.dumps(query, indent=2, separators=(',', ':'))
log.debug("Running query {}:\n{}".format(name, query_str))
yield query_activedata(query_str, config.url)
def __call__(self, taskgraph, label_to_taskid):
if not self.templates:
return taskgraph, label_to_taskid
for task in taskgraph.tasks.itervalues():
for template in sorted(self.templates):
context = {
'task': task.task,
'taskGroup': None,
'taskId': task.task_id,
'kind': task.kind,
'input': self.templates[template],
# The following context differs from action tasks
'attributes': task.attributes,
'label': task.label,
'target_tasks': self.target_tasks,
}
template = load_yaml(self.template_dir, template + '.yml')
result = jsone.render(template, context) or {}
for attr in ('task', 'attributes'):
if attr in result:
setattr(task, attr, result[attr])
return taskgraph, label_to_taskid
def run_query(name, args):
"""Loads and runs the specified query, yielding the result.
Given name of a query, this method will first read the query
from a .query file corresponding to the name.
After queries are loaded, each query to be run is inspected
and overridden if the provided context has values for limit.
The actual call to the ActiveData endpoint is encapsulated
inside the query_activedata method.
:param str name: name of the query file to be loaded.
:param Namespace args: namespace of ActiveData configs.
:return str: json-formatted string.
"""
context = vars(args)
query = load_query(name)
if 'limit' not in query and 'limit' in context:
query['limit'] = context['limit']
if 'format' not in query and 'format' in context:
query['format'] = context['format']
if config.debug:
query['meta'] = {"save": True}
query = jsone.render(query, context)
query_str = json.dumps(query, indent=2, separators=(',', ':'))
# translate "all" to a null value (which ActiveData will treat as all)
query_str = query_str.replace('"all"', 'null')
log.debug("Running query {}:\n{}".format(name, query_str))
return query_activedata(query_str, config.url)
def generate_beetmover_upstream_artifacts(job, platform, locale=None, dependencies=None):
"""Generate the upstream artifacts for beetmover, using the artifact map.
Currently only applies to beetmover tasks.
Args:
job (dict): The current job being generated
dependencies (list): A list of the job's dependency labels.
platform (str): The current build platform
locale (str): The current locale being beetmoved.
Returns:
list: A list of dictionaries conforming to the upstream_artifacts spec.
"""
base_artifact_prefix = get_artifact_prefix(job)
resolve_keyed_by(job, 'attributes.artifact_map', 'artifact map', platform=platform)
map_config = load_yaml(*os.path.split(job['attributes']['artifact_map']))
upstream_artifacts = list()
if not locale:
locales = map_config['default_locales']
else:
locales = [locale]
if not dependencies:
dependencies = job['dependencies'].keys()
for locale, dep in itertools.product(locales, dependencies):
paths = list()
for filename in map_config['mapping']:
if dep not in map_config['mapping'][filename]['from']:
continue
if locale != 'en-US' and not map_config['mapping'][filename]['all_locales']:
continue
# The next time we look at this file it might be a different locale.
file_config = deepcopy(map_config['mapping'][filename])
resolve_keyed_by(file_config, "source_path_modifier",
'source path modifier', locale=locale)
paths.append(os.path.join(
base_artifact_prefix,
jsone.render(file_config['source_path_modifier'], {'locale': locale}),
filename,
))
if not paths:
continue
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep)
},
"taskType": map_config['tasktype_map'].get(dep),
"paths": sorted(paths),
"locale": locale,
})
return upstream_artifacts
def render_action_hook(payload, context, delete_params=[]):
rendered_payload = jsone.render(payload, context)
# some parameters contain a lot of entries, so we hit the payload
# size limit. We don't use this parameter in any case, safe to
# remove
for param in delete_params:
del rendered_payload['decision']['parameters'][param]
return rendered_payload
def load_manifest(mtype, context={}):
"""Load the specified yaml file given the specified context"""
filename = f'{mtype}.yaml'
manifest_path = os.path.join(ROOT_REPO_DIR, 'k8s', filename)
with open(manifest_path) as mt:
manifest_template = yaml.safe_load(mt.read())
return jsone.render(manifest_template, context)
def test_verify_taskcluster_yml():
"""Verify that the json-e in the .taskcluster.yml is valid"""
with open(os.path.join(root, ".taskcluster.yml"), encoding="utf8") as f:
template = yaml.safe_load(f)
events = [("pr_event.json", "github-pull-request", "Pull Request"),
("master_push_event.json", "github-push", "Push to master")]
for filename, tasks_for, title in events:
with open(data_path(filename), encoding="utf8") as f:
event = json.load(f)
context = {"tasks_for": tasks_for,
"event": event,
"as_slugid": lambda x: x}
jsone.render(template, context)
def render_action_hook(payload, context, delete_params=[]):
rendered_payload = jsone.render(payload, context)
# some parameters contain a lot of entries, so we hit the payload
# size limit. We don't use this parameter in any case, safe to
# remove
for param in delete_params:
del rendered_payload['decision']['parameters'][param]
return rendered_payload
def test_hook_syntax(hook_path, payload):
"""
Validate the Taskcluster hook syntax
"""
assert os.path.exists(hook_path)
with open(hook_path, "r") as f:
# Patch the hook as in the taskboot deployment
content = f.read()
content = content.replace("REVISION", "deadbeef1234")
content = content.replace("CHANNEL", "test")
# Now parse it as json
hook_content = json.loads(content)
jsonschema.validate(instance=payload, schema=hook_content["triggerSchema"])
jsone.render(hook_content, context=payload)
def test():
exc, res = None, None
try:
res = jsone.render(spec['template'], spec['context'])
except Exception as e:
exc = e
if 'error' in spec:
assert exc, "expected exception"
else:
assert res == spec['result']
def run_query(name, **context):
for query in load_query(name):
# If limit is in the context, override the queries' value. We do this
# to keep the results down to a sane level when testing queries.
if 'limit' in context:
query['limit'] = context['limit']
query = jsone.render(query, context)
query_str = json.dumps(query, indent=2, separators=(',', ':'))
log.debug("Running query {}:\n{}".format(name, query_str))
yield query_activedata(query_str)
def write_file(template, context, suffix):
filepath = f"{args.destination}/{context['project_name']}-{suffix}.yaml"
try:
f = open(filepath, "x")
f.write(
yaml.dump(jsone.render(template, context),
default_flow_style=False,
width=float("inf")))
f.close()
except:
print(f"failed to write {filepath}")
def make_decision_task(params, root, symbol, arguments=[]):
"""Generate a basic decision task, based on the root .taskcluster.yml"""
with open(os.path.join(root, '.taskcluster.yml'), 'rb') as f:
taskcluster_yml = yaml.safe_load(f)
push_info = find_hg_revision_push_info(params['repository_url'],
params['head_rev'])
slugids = {}
def as_slugid(name):
# https://github.com/taskcluster/json-e/issues/164
name = name[0]
if name not in slugids:
slugids[name] = slugid.nice()
return slugids[name]
# provide a similar JSON-e context to what mozilla-taskcluster provides:
# https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
# but with a different tasks_for and an extra `cron` section
context = {
'tasks_for': 'cron',
'repository': {
'url': params['repository_url'],
'project': params['project'],
'level': params['level'],
},
'push': {
'revision': params['head_rev'],
# remainder are fake values, but the decision task expects them anyway
'pushlog_id': push_info['pushid'],
'pushdate': push_info['pushdate'],
'owner': 'cron',
'comment': '',
},
'cron': {
'task_id': os.environ.get('TASK_ID', '<cron task id>'),
'job_name': params['job_name'],
'job_symbol': symbol,
# args are shell-quoted since they are given to `bash -c`
'quoted_args': ' '.join(pipes.quote(a) for a in arguments),
},
'now': current_json_time(),
'as_slugid': as_slugid,
}
rendered = jsone.render(taskcluster_yml, context)
if len(rendered['tasks']) != 1:
raise Exception(
"Expected .taskcluster.yml to only produce one cron task")
task = rendered['tasks'][0]
task_id = task.pop('taskId')
return (task_id, task)
def try_ami(ami_id, tc_options):
pool = tc.parse_yaml("worker-pools.yml")["win2016"]
pool.pop("kind")
pool = tc.aws_windows(**pool)
pool = dict(description="", emailOnError=False, owner="*****@*****.**", **pool)
now = datetime.datetime.now().replace(microsecond=0).isoformat()
worker_type = "tmp-" + re.sub("[-:T]", "", now)
pool_id = "proj-servo/" + worker_type
task = {h["hookId"]: h for h in tc.parse_yaml("hooks.yml")}["daily"]["task"]
task["metadata"]["name"] = "Trying new Windows image " + ami_id
task["metadata"]["source"] = \
"https://github.com/servo/taskcluster-config/blob/master/commands/try-ami.py"
task["payload"]["env"]["SOURCE"] = task["metadata"]["source"]
task["payload"]["env"]["TASK_FOR"] = "try-windows-ami"
task["payload"]["env"]["GIT_REF"] = "refs/heads/try-windows-ami"
task["payload"]["env"]["NEW_AMI_WORKER_TYPE"] = worker_type
task["created"] = {"$eval": "now"}
task = jsone.render(task, {})
task_id = taskcluster.slugId()
wm = taskcluster.WorkerManager(tc_options)
queue = taskcluster.Queue(tc_options)
wm.createWorkerPool(pool_id, pool)
try:
queue.createTask(task_id, task)
task_view = "https://community-tc.services.mozilla.com/tasks/"
log("Created " + task_view + task_id)
while 1:
time.sleep(2)
result = queue.status(task_id)
state = result["status"]["state"]
if state not in ["unscheduled", "pending", "running"]:
log("Decision task:", state)
break
# The decision task has finished, so any other task should be scheduled now
while 1:
tasks = []
def handler(result):
for task in result["tasks"]:
if task["status"]["taskId"] != task_id:
tasks.append((task["status"]["taskId"], task["status"]["state"]))
queue.listTaskGroup(result["status"]["taskGroupId"], paginationHandler=handler)
if all(state not in ["unscheduled", "pending"] for _, state in tasks):
for task, _ in tasks:
log("Running " + task_view + task)
break
time.sleep(2)
finally:
wm.deleteWorkerPool(pool_id)
def generate_action_task(decision_task_id, action_task_input):
actions = fetch_actions_json(decision_task_id)
relpro = find_action("release-promotion", actions)
context = copy.deepcopy(actions["variables"]) # parameters
action_task_id = slugid.nice()
context.update({
"input": action_task_input,
"ownTaskId": action_task_id,
"taskId": None,
"task": None,
"taskGroupId": decision_task_id,
})
action_task = jsone.render(relpro["task"], context)
return action_task_id, action_task
def generate_action_task(decision_task_id, action_task_input):
actions = fetch_actions_json(decision_task_id)
relpro = find_action("release-promotion", actions)
context = copy.deepcopy(actions["variables"]) # parameters
action_task_id = slugid.nice()
context.update({
"input": action_task_input,
"ownTaskId": action_task_id,
"taskId": None,
"task": None,
"taskGroupId": decision_task_id,
})
action_task = jsone.render(relpro["task"], context)
return action_task_id, action_task
def main(worker_id_prefix, input, output):
"""Convert JSON/YAML templates into using json-e.
Accepts JSON or YAML format and outputs using JSON because it is YAML
compatible.
"""
config_template = yaml.safe_load(input)
context = os.environ.copy()
# special case for workerId, it must be unique and max 38 characters long,
# according to
# https://docs.taskcluster.net/docs/reference/platform/queue/api#declareWorker
worker_id = worker_id_prefix + slugid.nice().lower().replace("_", "").replace("-", "")
context["WORKER_ID"] = worker_id[:38]
config = jsone.render(config_template, context)
json.dump(config, output, indent=2, sort_keys=True)
def generate_action_task(decision_task_id, action_task_input):
actions = fetch_actions_json(decision_task_id)
relpro = find_action("release-promotion", actions)
context = copy.deepcopy(actions["variables"]) # parameters
action_task_id = slugid.nice()
context.update({
"input": action_task_input,
"ownTaskId": action_task_id,
"taskId": None,
"task": None,
})
action_task = jsone.render(relpro["task"], context)
# override ACTION_TASK_GROUP_ID, so we know the new ID in advance
action_task["payload"]["env"]["ACTION_TASK_GROUP_ID"] = action_task_id
return action_task_id, action_task
def make_decision_task(params):
"""Generate a basic decision task, based on the root .taskcluster.yml"""
with open(os.path.join(ROOT, '.taskcluster.yml'), 'rb') as f:
taskcluster_yml = yaml.safe_load(f)
slugids = {}
def as_slugid(name):
if name not in slugids:
slugids[name] = slugid.nice()
return slugids[name]
repository_parts = params['html_url'].split('/')
repository_full_name = '/'.join(
(repository_parts[-2], repository_parts[-1]))
# provide a similar JSON-e context to what taskcluster-github provides
context = {
'tasks_for': 'cron',
'cron': {
'task_id': params['cron_task_id'],
'name': params['name'],
},
'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
'as_slugid': as_slugid,
'event': {
'repository': {
'html_url': params['html_url'],
'full_name': repository_full_name,
},
'release': {
'tag_name': params['head_rev'],
'target_commitish': params['branch'],
},
'sender': {
'login': '******',
}
}
}
rendered = jsone.render(taskcluster_yml, context)
if len(rendered['tasks']) != 1:
raise Exception(
'Expected .taskcluster.yml to only produce one cron task')
task = rendered['tasks'][0]
task_id = task.pop('taskId')
return task_id, task
def generate_action_hook(decision_task_id, action_name, actions):
target_action = find_action(action_name, actions)
context = copy.deepcopy(actions['variables']) # parameters
context.update({
'input': {},
'taskGroupId': decision_task_id,
'taskId': None,
'task': None,
})
hook_payload = jsone.render(target_action['hookPayload'], context)
return dict(
hook_group_id=target_action['hookGroupId'],
hook_id=target_action['hookId'],
hook_payload=hook_payload,
context=context,
)
def test_jsone_validates(pipeline_file):
with open(pipeline_file, "r") as f:
yaml_content = yaml.safe_load(f.read())
result = jsone.render(yaml_content, context={"version": "42.0"})
tasks = result["tasks"]
all_ids = [task["ID"] for task in tasks]
# Make sure there are no duplicate IDs.
assert len(all_ids) == len(set(all_ids))
# Make sure all dependencies are present.
for task in tasks:
assert "dependencies" not in task or all(
dependency in all_ids for dependency in task["dependencies"])
def run(venv, **kwargs):
with open(os.path.join(root, ".taskcluster.yml")) as f:
template = yaml.safe_load(f)
events = [("pr_event.json", "github-pull-request", "Pull Request"),
("master_push_event.json", "github-push", "Push to master")]
for filename, tasks_for, title in events:
with open(os.path.join(here, "testdata", filename)) as f:
event = json.load(f)
context = {"tasks_for": tasks_for,
"event": event,
"as_slugid": lambda x: x}
data = jsone.render(template, context)
heading = "Got %s tasks for %s" % (len(data["tasks"]), title)
print(heading)
print("=" * len(heading))
for item in data["tasks"]:
print(json.dumps(item, indent=2))
print("")
def render_action_task(task, context):
action_task = jsone.render(task, context)
return action_task
