def install_helper(name):
helper = Helper.by_name(name)
filename = os.path.basename(helper.artifacts[0])
return [
'curl --compressed -o {} -L {{{}.artifacts[0]}}'.format(
filename, Helper.by_name(name)),
'chmod +x {}'.format(filename),
'git config --global cinnabar.helper $PWD/{}'.format(filename),
]
def main():
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
merge_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.zip -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcno-helper.zip -L {{{}.artifacts[1]}}'.format(task))
merge_coverage.append('(' + '& '.join(download_coverage) + '& wait)', )
for task in TestTask.coverage:
merge_coverage.extend([
'unzip -d cov-{{{}.id}} cov-{{{}.id}}.zip .coverage'.format(
task, task),
])
merge_coverage.extend([
'grcov -s repo -t lcov -o repo/coverage.lcov gcno-helper.zip ' +
' '.join('cov-{{{}.id}}.zip'.format(task)
for task in TestTask.coverage),
'cd repo',
'coverage combine --append {}'.format(' '.join(
'../cov-{{{}.id}}/.coverage'.format(task)
for task in TestTask.coverage)),
'cd ..',
])
if merge_coverage:
Task(
task_env=TaskEnvironment.by_name('linux.codecov'),
description='upload coverage',
scopes=['secrets:get:project/git-cinnabar/codecov'],
command=list(
chain(
Task.checkout(),
[
'set +x',
('export CODECOV_TOKEN=$(curl -sL '
'http://taskcluster/api/secrets/v1/secret/project/git-'
'cinnabar/codecov | '
'python -c "import json, sys; print(json.load(sys.stdin)'
'[\\"secret\\"][\\"token\\"])")'),
'set -x',
],
merge_coverage,
[
'cd repo',
'codecov --name "taskcluster" --commit {} --branch {}'.
format(TC_COMMIT, TC_BRANCH),
],
)),
)
for t in Task.by_id.values():
t.submit()
if not TC_ACTION and 'TC_GROUP_ID' in os.environ:
actions = {
'version': 1,
'actions': [],
'variables': {
'e': dict(TC_DATA, decision_id=''),
'tasks_for': 'action',
},
}
for name, a in action.by_name.items():
if name != 'decision':
actions['actions'].append({
'kind': 'task',
'name': a.name,
'title': a.title,
'description': a.description,
'context': [],
'task': a.task,
})
with open('actions.json', 'w') as out:
out.write(json.dumps(actions, indent=True))
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests &&'
' nosetests3 --all-modules tests)',
'(cd repo && python -m flake8 --ignore E402,F405'
' $(git ls-files \\*\\*.py git-cinnabar git-remote-hg'
' | grep -v ^CI/))',
'(cd repo && flake8 --ignore E402,F405'
' $(git ls-files CI/\\*\\*.py)'
' $(git grep -l unicode_literals))',
],
)
for env in ('linux', 'mingw64', 'osx10_10'):
# Can't spawn osx workers from pull requests.
if env.startswith('osx') and not TC_IS_PUSH:
continue
TestTask(task_env=env)
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(
chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
} if env == 'linux' else {},
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
clone=upgrade,
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
'GIT_CINNABAR_LOG': 'reexec:3',
'UPGRADE_FROM': upgrade,
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
TestTask(
task_env='linux',
variant='asan',
)
TestTask(
task_env='linux',
variant='asan',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
env={
'GIT_CINNABAR_OLD': '1',
},
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GIT_CINNABAR_OLD': '1',
'GRAFT': '1',
},
)
TestTask(
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
'GRAFT': '1',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
TestTask(
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
'GIT_CINNABAR_LOG': 'reexec:3',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
'GIT_CINNABAR_LOG': 'reexec:3',
'GRAFT': '1',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
for variant in ('coverage', 'old'):
env = {
'GIT_CINNABAR_CHECK': 'no-mercurial',
}
if variant == 'old':
env['GIT_CINNABAR_OLD_HELPER'] = '1'
TestTask(
variant=variant,
extra_desc='no-mercurial',
pre_command=[
'python -m virtualenv venv',
'. venv/bin/activate',
],
command=[
'deactivate',
# deactivate removes the git directory from $PATH.
# Also add the virtualenv bin directory to $PATH for mercurial
# to be found, but at the end for the system python to still
# be picked.
'export PATH=$PWD/git/bin:$PATH:$PWD/venv/bin',
'make -C repo -f CI/tests.mk',
],
env=env)
for variant in ('coverage', 'asan'):
for check in ([], ['no-mercurial']):
TestTask(
variant=variant,
extra_desc=' '.join(['cram'] + check),
clone=False,
command=[
'repo/git-cinnabar --version',
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK':
','.join(['no-version-check'] + check),
},
)
for check in ([], ['no-mercurial']):
TestTask(
extra_desc=' '.join(['cram'] + check),
clone=False,
command=[
'repo/git-cinnabar --version',
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK': ','.join(['no-version-check'] + check),
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os, task_env.cpu),
command=list(
chain(
install_git('{}.{}'.format(env, GIT_VERSION)),
install_hg('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
requests + [
'(cd repo ; ./git-cinnabar download)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the i686 windows helper, we need to manually touch
# it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests)',
'(cd repo && flake8 --ignore E402 $(git ls-files \*\*.py'
' git-cinnabar git-remote-hg))',
],
)
for env in ('linux', 'mingw64', 'osx10_10'):
TestTask(task_env=env)
for env in ('linux', 'mingw64', 'osx10_11'):
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
for env in ('linux', 'osx10_11'):
TestTask(
task_env=env,
variant='asan',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='coverage',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
upload_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.tar.xz -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcda-helper.tar.xz -L {{{}.artifacts[1]}}'.format(task))
upload_coverage.extend([
'(' + '& '.join(download_coverage) + '& wait)',
'tar -Jxf gcda-helper.tar.xz',
])
for task in TestTask.coverage:
upload_coverage.extend([
'tar -Jxf cov-{{{}.id}}.tar.xz'.format(task),
'codecov --name "{}" --commit {} --branch {}'.format(
task.task['metadata']['name'], TC_COMMIT, TC_BRANCH),
('find . \( -name .coverage -o -name coverage.xml -o -name \*.gcda'
' -o -name \*.gcov \) -delete'),
])
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests)',
'(cd repo && flake8 --ignore E402 $(git ls-files \\*\\*.py'
' git-cinnabar git-remote-hg | grep -v ^CI/))',
'(cd repo && python3 -m flake8 --ignore E402 '
'$(git ls-files CI/\\*\\*.py))',
],
)
TestTask(
description='cram tests',
variant='coverage',
clone=False,
command=[
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK': 'no-version-check',
},
)
for env in ('linux', 'mingw64', 'osx10_10'):
TestTask(task_env=env)
for env in ('linux', 'mingw64', 'osx10_11'):
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
for env in ('linux', 'osx10_11'):
TestTask(
task_env=env,
variant='asan',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
env={
'GIT_CINNABAR_OLD': '1',
},
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GIT_CINNABAR_OLD': '1',
'GRAFT': '1',
},
)
TestTask(
variant='coverage',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
for variant in ('coverage', 'old'):
env = {
'GIT_CINNABAR_CHECK': 'no-mercurial',
}
if variant == 'old':
env['GIT_CINNABAR_OLD_HELPER'] = '1'
TestTask(
variant=variant,
extra_desc='no-mercurial',
pre_command=[
'python -m virtualenv venv',
'. venv/bin/activate',
],
command=[
'deactivate',
# deactivate removes the git directory from $PATH.
# Also add the virtualenv bin directory to $PATH for mercurial
# to be found, but at the end for the system python to still
# be picked.
'export PATH=$PWD/git/bin:$PATH:$PWD/venv/bin',
'make -C repo -f CI/tests.mk',
],
env=env
)
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
upload_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.tar.xz -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcda-helper.tar.xz -L {{{}.artifacts[1]}}'.format(task))
upload_coverage.extend([
'(' + '& '.join(download_coverage) + '& wait)',
'tar -Jxf gcda-helper.tar.xz',
])
for task in TestTask.coverage:
upload_coverage.extend([
'tar -Jxf cov-{{{}.id}}.tar.xz'.format(task),
'codecov --name "{}" --commit {} --branch {}'.format(
task.task['metadata']['name'], TC_COMMIT, TC_BRANCH),
('find . \\( -name .coverage -o -name coverage.xml '
'-o -name \\*.gcda -o -name \\*.gcov \\) -delete'),
])