def make_task_graph(public_key,
signing_pvt_key,
product,
root_template="release_graph.yml.tmpl",
template_dir=DEFAULT_TEMPLATE_DIR,
**template_kwargs):
# TODO: some validation of template_kwargs + defaults
env = Environment(loader=FileSystemLoader([
path.join(template_dir, product),
path.join(template_dir, 'notification')
]),
undefined=StrictUndefined,
extensions=['jinja2.ext.do'])
now = arrow.now()
now_ms = now.timestamp * 1000
# Don't let the signing pvt key leak into the task graph.
with open(signing_pvt_key) as f:
pvt_key = f.read()
template = env.get_template(root_template)
template_vars = {
"product":
product,
"stableSlugId":
stableSlugId(),
"chunkify":
chunkify,
"sorted":
sorted,
"now":
now,
"now_ms":
now_ms,
# This is used in defining expirations in tasks. There's no way to
# actually tell Taskcluster never to expire them, but 1,000 years
# is as good as never....
"never":
arrow.now().replace(years=1000),
"pushlog_id":
get_json_rev(template_kwargs["repo_path"],
template_kwargs["revision"])["pushid"],
"get_treeherder_platform":
treeherder_platform,
"encrypt_env_var":
lambda *args: encryptEnvVar(*args, keyFile=public_key),
"buildbot2ftp":
buildbot2ftp,
"buildbot2bouncer":
buildbot2bouncer,
"sign_task":
partial(sign_task, pvt_key=pvt_key),
}
template_vars.update(template_kwargs)
return yaml.safe_load(template.render(**template_vars))
def test_repeat(text):
s = subject.stableSlugId()
assert s(text) == s(text)
def test_not_equal():
s = subject.stableSlugId()
assert s("first") != s("second")
@given(st.text())
def test_invalidate(text):
s1 = subject.stableSlugId()
s2 = subject.stableSlugId()
assert s1(text) != s2(text)
def make_task_graph(public_key, signing_pvt_key, product,
root_template="release_graph.yml.tmpl",
template_dir=DEFAULT_TEMPLATE_DIR,
**template_kwargs):
# TODO: some validation of template_kwargs + defaults
env = Environment(
loader=FileSystemLoader(path.join(template_dir, product)),
undefined=StrictUndefined,
extensions=['jinja2.ext.do'])
th = TreeherderClient()
now = arrow.now()
now_ms = now.timestamp * 1000
# Don't let the signing pvt key leak into the task graph.
with open(signing_pvt_key) as f:
pvt_key = f.read()
template = env.get_template(root_template)
template_vars = {
"product": product,
"stableSlugId": stableSlugId(),
"chunkify": chunkify,
"sorted": sorted,
"now": now,
"now_ms": now_ms,
# This is used in defining expirations in tasks. There's no way to
# actually tell Taskcluster never to expire them, but 1,000 years
# is as good as never....
"never": arrow.now().replace(years=1000),
# Treeherder expects 12 symbols in revision
"revision_hash": th.get_resultsets(
template_kwargs["branch"],
revision=template_kwargs["revision"][:12])[0]["revision_hash"],
"get_treeherder_platform": treeherder_platform,
"encrypt_env_var": lambda *args: encryptEnvVar(*args,
keyFile=public_key),
"buildbot2ftp": buildbot2ftp,
"buildbot2bouncer": buildbot2bouncer,
"sign_task": partial(sign_task, pvt_key=pvt_key),
}
template_vars.update(template_kwargs)
return yaml.safe_load(template.render(**template_vars))
def test_invalidate(text):
s1 = subject.stableSlugId()
s2 = subject.stableSlugId()
assert s1(text) != s2(text)
def test_not_equal():
s = subject.stableSlugId()
assert s("first") != s("second")
def test_invalidate(self, text):
s1 = subject.stableSlugId()
s2 = subject.stableSlugId()
self.assertNotEqual(s1(text), s2(text))
def test_not_equal(self):
s = subject.stableSlugId()
self.assertNotEqual(s("first"), s("second"))
def test_repeat(self, text):
s = subject.stableSlugId()
self.assertEqual(s(text), s(text))
def test_invalidate(self, text):
s1 = subject.stableSlugId()
s2 = subject.stableSlugId()
self.assertNotEqual(s1(text), s2(text))
def test_not_equal(self):
s = subject.stableSlugId()
self.assertNotEqual(s("first"), s("second"))
def test_repeat(self, text):
s = subject.stableSlugId()
self.assertEqual(s(text), s(text))
def main():
_idMaker = stableSlugId()
def idMaker(name):
return _idMaker(name).decode()
decisionTaskId = os.environ['TASK_ID']
owner = os.environ['GITHUB_HEAD_USER_EMAIL']
source = os.environ['GITHUB_HEAD_REPO_URL']
with requests.Session() as session:
for task in tasks:
dependencies = [
idMaker(name) for name in task.get('dependencies', [])
]
dependencies.append(decisionTaskId)
env = task.get('env', {})
for key, val in os.environ.items():
if key.startswith('GITHUB_'):
env.setdefault(key, val)
for spec in task.get('artifacts_from', []):
task_id = idMaker(spec['task_name'])
path = spec['path']
env[spec[
'env_var']] = f'{BASE_URL}/task/{task_id}/artifacts/{path}'
task_id = idMaker(task['name'])
res = session.put(
f'{BASE_URL}/task/{task_id}',
data=dumpJson({
'metadata': {
'name': task['name'],
'description': task['description'],
'owner': owner,
'source': source,
},
'provisionerId': 'aws-provisioner-v1',
'workerType': 'gecko-1-b-linux',
'schedulerId': 'taskcluster-github',
'taskGroupId': decisionTaskId,
'created': fromNow('0 seconds'),
'deadline': fromNow('1 day'),
'expires': fromNow('365 days'),
'payload': {
'image':
'mozilla/normandy-taskcluster:2017-07-26',
'command': [
'/bin/bash', '-c', ' && '.join([
'apt-get update',
'apt-get install -y git',
'mkdir /artifacts',
'cd ~',
'git clone $GITHUB_HEAD_REPO_URL normandy',
'pushd normandy',
'git checkout $GITHUB_HEAD_SHA',
'popd',
task['command'],
])
],
'maxRunTime':
28800, # 8 hours
'env':
env,
'artifacts': {
'public': {
'type': 'directory',
'path': '/artifacts',
'expires':
fromNow('364 days'), # must expire before task
},
},
'features': {
'taskclusterProxy': True,
},
},
'dependencies': dependencies,
}))
print(res.text)
res.raise_for_status()
"platforms": PLATFORMS, "locales": common_locales, "chunks": 10, "from_version": FROM["version"], "from_build_number": FROM["build_number"], "to_version": TO["version"], "to_build_number": TO["build_number"], "stableSlugId": stableSlugId(), "chunkify": chunkify, "sorted": sorted, "now": now, "now_ms": now_ms, "never": arrow.now().replace(years=1), "encrypt_env_var": lambda *args: encryptEnvVar(*args, keyFile='docker-worker-pub.pem'), "sign_task": partial(sign_task, pvt_key=pvt_key), "balrog_username":
