def __init__(self, os_and_version):
(os, version) = os_and_version.split('.', 1)
if os.startswith('osx'):
build_image = TaskEnvironment.by_name('osx10_10.build')
else:
build_image = DockerImage.by_name('build')
if os == 'linux' or os.startswith('osx'):
h = hashlib.sha1(build_image.hexdigest)
h.update('v2')
if os == 'linux':
description = 'git v{}'.format(version)
else:
env = build_image
description = 'git v{} {} {}'.format(version, env.os, env.cpu)
Task.__init__(
self,
task_env=build_image,
description=description,
index='{}.git.v{}'.format(h.hexdigest(), version),
expireIn='26 weeks',
command=Task.checkout(
'git://git.kernel.org/pub/scm/git/git.git',
'v{}'.format(version)) +
[
'make -C repo -j$(nproc) install prefix=/'
' NO_GETTEXT=1 NO_OPENSSL=1 NO_TCLTK=1'
' DESTDIR=$PWD/git',
'tar -Jcf $ARTIFACTS/git-{}.tar.xz git'.format(version),
],
artifact='git-{}.tar.xz'.format(version),
)
else:
env = TaskEnvironment.by_name('{}.build'.format(os))
raw_version = version
if 'windows' not in version:
version = {
version: version + '.windows.1',
'2.17.1': '2.17.1.windows.2',
}.get(version)
if version.endswith('.windows.1'):
min_ver = version[:-len('.windows.1')]
else:
min_ver = version.replace('windows.', '')
Task.__init__(
self,
task_env=build_image,
description='git v{} {} {}'.format(version, env.os, env.cpu),
index='{}.git.v{}'.format(os, raw_version),
expireIn='26 weeks',
command=[
'curl -L https://github.com/git-for-windows/git/releases/'
'download/v{}/MinGit-{}-{}-bit.zip'
' -o git.zip'.format(version, min_ver, msys.bits(env.cpu)),
'unzip -d git git.zip',
'tar -jcf $ARTIFACTS/git-{}.tar.bz2 git'.format(
raw_version),
],
artifact='git-{}.tar.bz2'.format(raw_version),
)
def __init__(self, os_and_version):
(os, version) = os_and_version.split('.', 1)
env = TaskEnvironment.by_name('{}.build'.format(os))
desc = 'hg v{}'.format(version)
if os == 'linux':
artifact = 'mercurial-{}-cp27-none-linux_x86_64.whl'
else:
desc = '{} {} {}'.format(desc, env.os, env.cpu)
artifact = 'mercurial-{}-cp27-cp27m-mingw.whl'
# 2.6.2 is the first version available on pypi
if parse_version('2.6.2') <= parse_version(version):
source = 'mercurial=={}'
else:
source = 'https://mercurial-scm.org/release/mercurial-{}.tar.gz'
Task.__init__(
self,
task_env=env,
description=desc,
index='{}.hg.v{}'.format(env.hexdigest, version),
expireIn='26 weeks',
command=(
'pip wheel -v --build-option -b --build-option $PWD/wheel'
' -w $ARTIFACTS {}'.format(source.format(version)),
),
artifact=artifact.format(version),
)
def __init__(self, os_and_variant):
os, variant = (os_and_variant.split('.', 2) + [''])[:2]
env = TaskEnvironment.by_name('{}.build'.format(os))
artifact = 'git-cinnabar-helper'
if os != 'linux':
artifact += '.exe'
artifacts = [artifact]
def prefix(p, s):
return p + s if s else s
make_flags = []
hash = None
head = None
desc_variant = variant
extra_commands = []
if variant == 'asan':
make_flags.append('CFLAGS="-O2 -g -fsanitize=address"')
make_flags.append('LDFLAGS=-static-libasan')
elif variant == 'coverage':
make_flags.append('CFLAGS="-coverage"')
artifacts += ['coverage.tar.xz']
extra_commands = [
'mv repo/git-core/{{cinnabar,connect,hg}}*.gcno repo/helper',
'(cd repo && tar -Jcf $ARTIFACTS/coverage.tar.xz'
' helper/{{cinnabar,connect,hg}}*.gcno)',
]
elif variant == 'old':
head = old_helper_head()
hash = old_helper_hash(head)
variant = ''
elif variant:
raise Exception('Unknown variant: {}'.format(variant))
if os == 'linux':
make_flags.append('CURL_COMPAT=1')
else:
make_flags.append('USE_LIBPCRE1=YesPlease')
make_flags.append('USE_LIBPCRE2=')
make_flags.append('CFLAGS+=-DCURLOPT_PROXY_CAINFO=246')
hash = hash or helper_hash()
Task.__init__(
self,
task_env=env,
description='helper {} {}{}'.format(
env.os, env.cpu, prefix(' ', desc_variant)),
index='helper.{}.{}.{}{}'.format(
hash, env.os, env.cpu, prefix('.', variant)),
expireIn='26 weeks',
command=Task.checkout(commit=head) + [
'make -C repo -j $(nproc) helper prefix=/usr{}'.format(
prefix(' ', ' '.join(make_flags))),
'mv repo/{} $ARTIFACTS/'.format(artifact),
] + extra_commands,
artifacts=artifacts,
)
def __init__(self, os_and_version):
(os, version) = os_and_version.split('.', 1)
env = TaskEnvironment.by_name('{}.build'.format(os))
kwargs = {}
if len(version) == 40:
# Assume it's a sha1
pretty_version = 'r{}'.format(version)
artifact_version = 'unknown'
expire = '2 weeks'
else:
pretty_version = 'v{}'.format(version)
artifact_version = version
expire = '26 weeks'
desc = 'hg {}'.format(pretty_version)
if os == 'linux':
artifact = 'mercurial-{}-cp27-none-linux_x86_64.whl'
else:
desc = '{} {} {}'.format(desc, env.os, env.cpu)
if os.startswith('osx'):
artifact = ('mercurial-{{}}-cp27-cp27m-macosx_{}_intel.whl'
.format(os[3:]))
kwargs.setdefault('env', {})['MACOSX_DEPLOYMENT_TARGET'] = \
'10.10'
else:
artifact = 'mercurial-{}-cp27-cp27m-mingw.whl'
pre_command = []
if len(version) == 40:
source = './hg'
pre_command.extend(
self.install('{}.{}'.format(os, MERCURIAL_VERSION)))
pre_command.extend([
'hg clone https://www.mercurial-scm.org/repo/hg -r {}'
.format(version),
'rm -rf hg/.hg',
])
# 2.6.2 is the first version available on pypi
elif parse_version('2.6.2') <= parse_version(version):
source = 'mercurial=={}'
else:
source = 'https://mercurial-scm.org/release/mercurial-{}.tar.gz'
Task.__init__(
self,
task_env=env,
description=desc,
index='{}.hg.{}'.format(env.hexdigest, pretty_version),
expireIn=expire,
command=pre_command + [
'python -m pip wheel -v --build-option -b --build-option'
' $PWD/wheel -w $ARTIFACTS {}'.format(source.format(version)),
],
artifact=artifact.format(artifact_version),
**kwargs
)
def __init__(self, os_and_version):
(os, version) = os_and_version.split('.', 1)
env = TaskEnvironment.by_name('{}.build'.format(os))
kwargs = {}
if len(version) == 40:
# Assume it's a sha1
pretty_version = 'r{}'.format(version)
artifact_version = 'unknown'
expire = '2 weeks'
else:
pretty_version = 'v{}'.format(version)
artifact_version = version
expire = '26 weeks'
desc = 'hg {}'.format(pretty_version)
if os == 'linux':
artifact = 'mercurial-{}-cp27-none-linux_x86_64.whl'
else:
desc = '{} {} {}'.format(desc, env.os, env.cpu)
if os.startswith('osx'):
artifact = ('mercurial-{{}}-cp27-cp27m-macosx_{}_intel.whl'
.format(os[3:]))
kwargs.setdefault('env', {})['MACOSX_DEPLOYMENT_TARGET'] = \
'10.10'
else:
artifact = 'mercurial-{}-cp27-cp27m-mingw.whl'
pre_command = []
if len(version) == 40:
source = './hg'
pre_command.extend(
self.install('{}.{}'.format(os, MERCURIAL_VERSION)))
pre_command.extend([
'hg clone https://www.mercurial-scm.org/repo/hg -r {}'
.format(version),
'rm -rf hg/.hg',
])
# 2.6.2 is the first version available on pypi
elif parse_version('2.6.2') <= parse_version(version):
source = 'mercurial=={}'
else:
source = 'https://mercurial-scm.org/release/mercurial-{}.tar.gz'
Task.__init__(
self,
task_env=env,
description=desc,
index='{}.hg.{}'.format(env.hexdigest, pretty_version),
expireIn=expire,
command=pre_command + [
'python -m pip wheel -v --build-option -b --build-option'
' $PWD/wheel -w $ARTIFACTS {}'.format(source.format(version)),
],
artifact=artifact.format(artifact_version),
**kwargs
)
def __init__(self, os_and_version):
(os, version) = os_and_version.split('.', 1)
(version, suffix, _) = version.partition('.py3')
if suffix:
python = 'python3'
else:
python = 'python'
env = TaskEnvironment.by_name('{}.build'.format(os))
kwargs = {}
if len(version) == 40:
# Assume it's a sha1
pretty_version = 'r{}{}'.format(version, suffix)
artifact_version = 'unknown'
expire = '2 weeks'
else:
pretty_version = 'v{}{}'.format(version, suffix)
artifact_version = version
expire = '26 weeks'
desc = 'hg {}'.format(pretty_version)
if os == 'linux':
if python == 'python3':
artifact = 'mercurial-{}-cp35-cp35m-linux_x86_64.whl'
else:
artifact = 'mercurial-{}-cp27-cp27mu-linux_x86_64.whl'
else:
desc = '{} {} {}'.format(desc, env.os, env.cpu)
if os.startswith('osx'):
wheel_cpu = 'x86_64'
artifact = (
'mercurial-{{}}-cp27-cp27m-macosx_{}_{}.whl'.format(
env.os_version.replace('.', '_'), wheel_cpu))
else:
artifact = 'mercurial-{}-cp27-cp27m-mingw.whl'
pre_command = []
if len(version) == 40:
source = './hg'
pre_command.extend(
self.install('{}.{}'.format(os, MERCURIAL_VERSION)))
pre_command.extend([
'hg clone https://www.mercurial-scm.org/repo/hg -r {}'.format(
version),
'rm -rf hg/.hg',
])
# 2.6.2 is the first version available on pypi
elif parse_version('2.6.2') <= parse_version(version):
source = 'mercurial=={}'
else:
source = 'https://mercurial-scm.org/release/mercurial-{}.tar.gz'
h = hashlib.sha1(env.hexdigest.encode())
h.update(artifact.encode())
Task.__init__(self,
task_env=env,
description=desc,
index='{}.hg.{}'.format(h.hexdigest(), pretty_version),
expireIn=expire,
command=pre_command + [
'{} -m pip wheel -v --build-option -b --build-option'
' $PWD/wheel -w $ARTIFACTS {}'.format(
python, source.format(version)),
],
artifact=artifact.format(artifact_version),
**kwargs)
def __init__(self, **kwargs):
git = kwargs.pop('git', GIT_VERSION)
hg = kwargs.pop('hg', MERCURIAL_VERSION)
commit = kwargs.pop('commit', None)
task_env = kwargs.pop('task_env', 'linux')
variant = kwargs.pop('variant', None)
helper = kwargs.pop('helper', None)
clone = kwargs.pop('clone', GITHUB_HEAD_SHA)
desc = kwargs.pop('description', None)
extra_desc = kwargs.pop('extra_desc', None)
if helper is None:
helper = '{}.{}'.format(task_env, variant) if variant else task_env
helper = install_helper(helper)
if variant:
kwargs.setdefault('env', {})['VARIANT'] = variant
env = TaskEnvironment.by_name('{}.test'.format(task_env))
command = []
if hg:
command.extend(install_hg('{}.{}'.format(task_env, hg)))
if git:
command.extend(install_git('{}.{}'.format(task_env, git)))
command.extend(Task.checkout(commit=commit))
command.extend(helper)
if clone:
command.extend([
'curl -L {{{}.artifact}} -o clone.tar.xz'.format(
Clone.by_name(clone)),
'tar -C repo -Jxf clone.tar.xz',
])
if 'command' in kwargs:
kwargs['command'] = command + kwargs['command']
else:
kwargs['command'] = command + [
'make -C repo -f CI.mk script',
'make -C repo -f CI.mk script NO_BUNDLE2=1 UPGRADE_FROM=',
]
if variant == 'coverage':
kwargs['command'].extend([
'shopt -s nullglob',
'for f in repo/git-core/{{cinnabar,connect,hg}}*.gcda',
'do mv $f repo/helper',
'done',
'cd repo',
'tar -Jcf $ARTIFACTS/coverage.tar.xz .coverage'
' helper/{{cinnabar,connect,hg}}*.gcda',
'cd ..',
'shopt -u nullglob',
])
artifact = kwargs.pop('artifact', None)
artifacts = kwargs.setdefault('artifacts', [])
assert not (artifacts and artifact)
if artifact:
artifacts.push(artifact)
artifacts.append('coverage.tar.xz')
self.coverage.append(self)
if not desc:
desc = 'test w/ git-{} hg-{}'.format(git, hg)
if variant and variant != 'coverage':
desc = ' '.join((desc, variant))
if extra_desc:
desc = ' '.join((desc, extra_desc))
if task_env != 'linux':
desc = ' '.join((desc, env.os, env.cpu))
kwargs['description'] = desc
Task.__init__(self, task_env=env, **kwargs)
},
)
TestTask(description='python lint & tests',
variant='coverage',
clone=False,
env={
'PYTHON_CHECKS': '1',
})
for env in ('linux', 'mingw64'):
TestTask(task_env=env)
requests = [] if env == 'linux' else ['pip install requests']
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os, task_env.cpu),
command=list(
chain(
install_git('{}.{}'.format(env, GIT_VERSION)),
install_hg('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
requests + [
'(cd repo ; ./git-cinnabar download)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests)',
'(cd repo && flake8 --ignore E402 $(git ls-files \*\*.py'
' git-cinnabar git-remote-hg))',
],
)
for env in ('linux', 'mingw64', 'osx10_10'):
TestTask(task_env=env)
for env in ('linux', 'mingw64', 'osx10_11'):
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
for env in ('linux', 'osx10_11'):
TestTask(
task_env=env,
variant='asan',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='coverage',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests)',
'(cd repo && flake8 --ignore E402 $(git ls-files \\*\\*.py'
' git-cinnabar git-remote-hg | grep -v ^CI/))',
'(cd repo && python3 -m flake8 --ignore E402 '
'$(git ls-files CI/\\*\\*.py))',
],
)
TestTask(
description='cram tests',
variant='coverage',
clone=False,
command=[
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK': 'no-version-check',
},
)
for env in ('linux', 'mingw64', 'osx10_10'):
TestTask(task_env=env)
for env in ('linux', 'mingw64', 'osx10_11'):
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
for env in ('linux', 'osx10_11'):
TestTask(
task_env=env,
variant='asan',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
env={
'GIT_CINNABAR_OLD': '1',
},
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GIT_CINNABAR_OLD': '1',
'GRAFT': '1',
},
)
TestTask(
variant='coverage',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
for variant in ('coverage', 'old'):
env = {
'GIT_CINNABAR_CHECK': 'no-mercurial',
}
if variant == 'old':
env['GIT_CINNABAR_OLD_HELPER'] = '1'
TestTask(
variant=variant,
extra_desc='no-mercurial',
pre_command=[
'python -m virtualenv venv',
'. venv/bin/activate',
],
command=[
'deactivate',
# deactivate removes the git directory from $PATH.
# Also add the virtualenv bin directory to $PATH for mercurial
# to be found, but at the end for the system python to still
# be picked.
'export PATH=$PWD/git/bin:$PATH:$PWD/venv/bin',
'make -C repo -f CI/tests.mk',
],
env=env
)
def main():
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
merge_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.zip -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcno-helper.zip -L {{{}.artifacts[1]}}'.format(task))
merge_coverage.append('(' + '& '.join(download_coverage) + '& wait)', )
for task in TestTask.coverage:
merge_coverage.extend([
'unzip -d cov-{{{}.id}} cov-{{{}.id}}.zip .coverage'.format(
task, task),
])
merge_coverage.extend([
'grcov -s repo -t lcov -o repo/coverage.lcov gcno-helper.zip ' +
' '.join('cov-{{{}.id}}.zip'.format(task)
for task in TestTask.coverage),
'cd repo',
'coverage combine --append {}'.format(' '.join(
'../cov-{{{}.id}}/.coverage'.format(task)
for task in TestTask.coverage)),
'cd ..',
])
if merge_coverage:
Task(
task_env=TaskEnvironment.by_name('linux.codecov'),
description='upload coverage',
scopes=['secrets:get:project/git-cinnabar/codecov'],
command=list(
chain(
Task.checkout(),
[
'set +x',
('export CODECOV_TOKEN=$(curl -sL '
'http://taskcluster/api/secrets/v1/secret/project/git-'
'cinnabar/codecov | '
'python -c "import json, sys; print(json.load(sys.stdin)'
'[\\"secret\\"][\\"token\\"])")'),
'set -x',
],
merge_coverage,
[
'cd repo',
'codecov --name "taskcluster" --commit {} --branch {}'.
format(TC_COMMIT, TC_BRANCH),
],
)),
)
for t in Task.by_id.values():
t.submit()
if not TC_ACTION and 'TC_GROUP_ID' in os.environ:
actions = {
'version': 1,
'actions': [],
'variables': {
'e': dict(TC_DATA, decision_id=''),
'tasks_for': 'action',
},
}
for name, a in action.by_name.items():
if name != 'decision':
actions['actions'].append({
'kind': 'task',
'name': a.name,
'title': a.title,
'description': a.description,
'context': [],
'task': a.task,
})
with open('actions.json', 'w') as out:
out.write(json.dumps(actions, indent=True))
def __init__(self, os_and_variant):
os, variant = (os_and_variant.split('.', 2) + [''])[:2]
if variant == 'asan' and os == 'osx10_10':
os = 'osx10_11'
env = TaskEnvironment.by_name('{}.build'.format(os))
artifact = 'git-cinnabar-helper'
if os.startswith('mingw'):
artifact += '.exe'
artifacts = [artifact]
def prefix(p, s):
return p + s if s else s
make_flags = []
hash = None
head = None
desc_variant = variant
extra_commands = []
if variant == 'asan':
if os.startswith('osx'):
opt = '-O2'
else:
opt = '-Og'
make_flags.append('LDFLAGS=-static-libasan')
make_flags.append(
'CFLAGS="{} -g -fsanitize=address -fno-omit-frame-pointer"'
.format(opt))
elif variant == 'coverage':
make_flags.append('CFLAGS="-coverage"')
artifacts += ['coverage.tar.xz']
extra_commands = [
'mv repo/git-core/{{cinnabar,connect,hg}}*.gcno repo/helper',
'(cd repo && tar -Jcf $ARTIFACTS/coverage.tar.xz'
' helper/{{cinnabar,connect,hg}}*.gcno)',
]
elif variant == 'old' or variant.startswith('old:'):
if len(variant) > 3:
head = variant[4:]
else:
head = old_helper_head()
hash = helper_hash(head)
variant = ''
elif variant:
raise Exception('Unknown variant: {}'.format(variant))
if os == 'linux':
make_flags.append('CURL_COMPAT=1')
elif not os.startswith('osx'):
make_flags.append('USE_LIBPCRE1=YesPlease')
make_flags.append('USE_LIBPCRE2=')
make_flags.append('CFLAGS+=-DCURLOPT_PROXY_CAINFO=246')
hash = hash or helper_hash()
Task.__init__(
self,
task_env=env,
description='helper {} {}{}'.format(
env.os, env.cpu, prefix(' ', desc_variant)),
index='helper.{}.{}.{}{}'.format(
hash, env.os, env.cpu, prefix('.', variant)),
expireIn='26 weeks',
command=Task.checkout(commit=head) + [
'make -C repo helper -j $({}) prefix=/usr{} V=1'.format(
nproc(env), prefix(' ', ' '.join(make_flags))),
'mv repo/{} $ARTIFACTS/'.format(artifact),
] + extra_commands,
artifacts=artifacts,
)
def __init__(self, **kwargs):
git = kwargs.pop('git', GIT_VERSION)
hg = kwargs.pop('hg', MERCURIAL_VERSION)
commit = kwargs.pop('commit', None)
task_env = kwargs.pop('task_env', 'linux')
variant = kwargs.pop('variant', None)
helper = kwargs.pop('helper', None)
clone = kwargs.pop('clone', TC_COMMIT)
desc = kwargs.pop('description', None)
extra_desc = kwargs.pop('extra_desc', None)
pre_command = kwargs.pop('pre_command', None)
if helper is None:
helper = '{}.{}'.format(task_env, variant) if variant else task_env
helper = Helper.install(helper)
if variant:
kwargs.setdefault('env', {})['VARIANT'] = variant
env = TaskEnvironment.by_name('{}.test'.format(task_env))
command = []
if pre_command:
command.extend(pre_command)
if hg:
command.extend(Hg.install('{}.{}'.format(task_env, hg)))
command.append('hg --version')
if LooseVersion(hg) < '3.6':
kwargs.setdefault('env', {})['NO_CLONEBUNDLES'] = '1'
if git:
command.extend(Git.install('{}.{}'.format(task_env, git)))
command.append('git --version')
command.extend(Task.checkout(commit=commit))
command.extend(helper)
if clone:
command.extend([
'curl -L {{{}.artifact}} -o repo/bundle.git'.format(
Clone.by_name(clone)),
'git init repo/hg.old.git',
'git -C repo/hg.old.git fetch ../bundle.git refs/*:refs/*',
'git -C repo/hg.old.git remote add origin hg::$REPO',
'git -C repo/hg.old.git symbolic-ref HEAD'
' refs/heads/branches/default/tip',
])
kwargs.setdefault('env', {})['REPO'] = REPO
if 'command' in kwargs:
kwargs['command'] = command + kwargs['command']
else:
if commit:
# Always use the current CI scripts
command.append('git -C repo checkout {} CI'.format(TC_COMMIT))
kwargs['command'] = command + [
'make -C repo -f CI/tests.mk',
]
if variant == 'coverage':
kwargs['command'].extend([
'shopt -s nullglob',
'for f in repo/git-core/{{cinnabar,connect,hg}}*.gcda',
'do mv $f repo/helper',
'done',
'cd repo',
'tar -Jcf $ARTIFACTS/coverage.tar.xz .coverage'
' helper/{{cinnabar,connect,hg}}*.gcda',
'cd ..',
'shopt -u nullglob',
])
artifact = kwargs.pop('artifact', None)
artifacts = kwargs.setdefault('artifacts', [])
assert not(artifacts and artifact)
if artifact:
artifacts.push(artifact)
artifacts.append('coverage.tar.xz')
self.coverage.append(self)
if not desc:
desc = 'test w/ git-{} hg-{}'.format(
git, 'r' + hg if len(hg) == 40 else hg)
if variant and variant != 'coverage':
desc = ' '.join((desc, variant))
if extra_desc:
desc = ' '.join((desc, extra_desc))
if task_env != 'linux':
desc = ' '.join((desc, env.os, env.cpu))
kwargs['description'] = desc
Task.__init__(self, task_env=env, **kwargs)
def __init__(self, os_and_variant):
os, variant = (os_and_variant.split('.', 2) + [''])[:2]
if os.startswith('osx'):
os = 'osx'
env = TaskEnvironment.by_name('{}.build'.format(os))
artifact = 'git-cinnabar-helper'
if os.startswith('mingw'):
artifact += '.exe'
artifacts = [artifact]
def prefix(p, s):
return p + s if s else s
make_flags = []
hash = None
head = None
desc_variant = variant
extra_commands = []
if variant == 'asan':
if os.startswith('osx'):
opt = '-O2'
else:
opt = '-Og'
make_flags.append('LDFLAGS=-static-libasan')
make_flags.append(
'CFLAGS="{} -g -fsanitize=address -fno-omit-frame-pointer"'.
format(opt))
elif variant == 'coverage':
make_flags.append('CFLAGS="-coverage"')
artifacts += ['coverage.zip']
extra_commands = [
'mv repo/git-core/{{cinnabar,connect,hg}}*.gcno repo/helper',
'(cd repo && zip $ARTIFACTS/coverage.zip'
' helper/{{cinnabar,connect,hg}}*.gcno)',
]
elif variant == 'old' or variant.startswith('old:'):
if len(variant) > 3:
head = variant[4:]
else:
head = old_helper_head()
hash = helper_hash(head)
variant = ''
elif variant:
raise Exception('Unknown variant: {}'.format(variant))
if os == 'linux':
make_flags.append('CURL_COMPAT=1')
elif os == 'arm64-osx':
make_flags.append('CFLAGS+="-arch {}"'.format(env.cpu))
make_flags.append('LDFLAGS+="-arch {}"'.format(env.cpu))
elif not os.startswith('osx'):
make_flags.append('USE_LIBPCRE1=YesPlease')
make_flags.append('USE_LIBPCRE2=')
make_flags.append('CFLAGS+=-DCURLOPT_PROXY_CAINFO=246')
hash = hash or helper_hash()
kwargs = {}
if os.startswith('osx'):
kwargs.setdefault('env', {}).setdefault('MACOSX_DEPLOYMENT_TARGET',
'10.7')
Task.__init__(
self,
task_env=env,
description='helper {} {}{}'.format(env.os, env.cpu,
prefix(' ', desc_variant)),
index='helper.{}.{}.{}{}'.format(hash, env.os, env.cpu,
prefix('.', variant)),
expireIn='26 weeks',
command=Task.checkout(commit=head) + [
'make -C repo helper -j $({}) prefix=/usr{} V=1'.format(
nproc(env), prefix(' ', ' '.join(make_flags))),
'mv repo/{} $ARTIFACTS/'.format(artifact),
] + extra_commands,
artifacts=artifacts,
**kwargs,
)
def __init__(self, os_and_version):
(os, version) = os_and_version.split('.', 1)
if os.startswith('osx'):
build_image = TaskEnvironment.by_name('osx10_10.build')
else:
build_image = DockerImage.by_name('build')
if os == 'linux' or os.startswith('osx'):
h = hashlib.sha1(build_image.hexdigest.encode())
h.update(b'v2')
if os == 'linux':
description = 'git v{}'.format(version)
else:
env = build_image
description = 'git v{} {} {}'.format(version, env.os, env.cpu)
Task.__init__(
self,
task_env=build_image,
description=description,
index='{}.git.v{}'.format(h.hexdigest(), version),
expireIn='26 weeks',
command=Task.checkout(
'git://git.kernel.org/pub/scm/git/git.git',
'v{}'.format(version)
) + [
'make -C repo -j$({}) install prefix=/ NO_GETTEXT=1'
' NO_OPENSSL=1 NO_TCLTK=1 DESTDIR=$PWD/git'.format(
nproc(build_image)),
'tar -Jcf $ARTIFACTS/git-{}.tar.xz git'
.format(version),
],
artifact='git-{}.tar.xz'.format(version),
)
else:
env = TaskEnvironment.by_name('{}.build'.format(os))
raw_version = version
if 'windows' not in version:
version = {
version: version + '.windows.1',
'2.17.1': '2.17.1.windows.2',
}.get(version)
if version.endswith('.windows.1'):
min_ver = version[:-len('.windows.1')]
else:
min_ver = version.replace('windows.', '')
Task.__init__(
self,
task_env=build_image,
description='git v{} {} {}'.format(version, env.os, env.cpu),
index='{}.git.v{}'.format(os, raw_version),
expireIn='26 weeks',
command=[
'curl -L https://github.com/git-for-windows/git/releases/'
'download/v{}/MinGit-{}-{}-bit.zip'
' -o git.zip'.format(version, min_ver, msys.bits(env.cpu)),
'unzip -d git git.zip',
'curl -L https://github.com/git-for-windows/git/releases/'
'download/v{}/Git-{}-{}-bit.tar.bz2 | '
'tar -C git -jx {}/libexec/git-core/git-http-backend.exe'
.format(version, min_ver, msys.bits(env.cpu),
msys.mingw(env.cpu).lower()),
'tar -jcf $ARTIFACTS/git-{}.tar.bz2 git'.format(
raw_version),
],
artifact='git-{}.tar.bz2'.format(raw_version),
)
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests &&'
' nosetests3 --all-modules tests)',
'(cd repo && python -m flake8 --ignore E402,F405'
' $(git ls-files \\*\\*.py git-cinnabar git-remote-hg'
' | grep -v ^CI/))',
'(cd repo && flake8 --ignore E402,F405'
' $(git ls-files CI/\\*\\*.py)'
' $(git grep -l unicode_literals))',
],
)
for env in ('linux', 'mingw64', 'osx10_10'):
# Can't spawn osx workers from pull requests.
if env.startswith('osx') and not TC_IS_PUSH:
continue
TestTask(task_env=env)
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(
chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
} if env == 'linux' else {},
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
clone=upgrade,
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
'GIT_CINNABAR_LOG': 'reexec:3',
'UPGRADE_FROM': upgrade,
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
TestTask(
task_env='linux',
variant='asan',
)
TestTask(
task_env='linux',
variant='asan',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
env={
'GIT_CINNABAR_OLD': '1',
},
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GIT_CINNABAR_OLD': '1',
'GRAFT': '1',
},
)
TestTask(
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
'GRAFT': '1',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
TestTask(
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
'GIT_CINNABAR_LOG': 'reexec:3',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
'GIT_CINNABAR_LOG': 'reexec:3',
'GRAFT': '1',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
for variant in ('coverage', 'old'):
env = {
'GIT_CINNABAR_CHECK': 'no-mercurial',
}
if variant == 'old':
env['GIT_CINNABAR_OLD_HELPER'] = '1'
TestTask(
variant=variant,
extra_desc='no-mercurial',
pre_command=[
'python -m virtualenv venv',
'. venv/bin/activate',
],
command=[
'deactivate',
# deactivate removes the git directory from $PATH.
# Also add the virtualenv bin directory to $PATH for mercurial
# to be found, but at the end for the system python to still
# be picked.
'export PATH=$PWD/git/bin:$PATH:$PWD/venv/bin',
'make -C repo -f CI/tests.mk',
],
env=env)
for variant in ('coverage', 'asan'):
for check in ([], ['no-mercurial']):
TestTask(
variant=variant,
extra_desc=' '.join(['cram'] + check),
clone=False,
command=[
'repo/git-cinnabar --version',
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK':
','.join(['no-version-check'] + check),
},
)
for check in ([], ['no-mercurial']):
TestTask(
extra_desc=' '.join(['cram'] + check),
clone=False,
command=[
'repo/git-cinnabar --version',
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK': ','.join(['no-version-check'] + check),
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
upload_coverage.extend([
'(' + '& '.join(download_coverage) + '& wait)',
'tar -Jxf gcda-helper.tar.xz',
])
for task in TestTask.coverage:
upload_coverage.extend([
'tar -Jxf cov-{{{}.id}}.tar.xz'.format(task),
'codecov --name "{}" --commit {} --branch {}'.format(
task.task['metadata']['name'], TC_COMMIT, TC_BRANCH),
('find . \( -name .coverage -o -name coverage.xml -o -name \*.gcda'
' -o -name \*.gcov \) -delete'),
])
if upload_coverage:
Task(
task_env=TaskEnvironment.by_name('linux.codecov'),
description='upload coverage',
scopes=['secrets:get:repo:github.com/glandium.git-cinnabar:codecov'],
command=list(chain(
Task.checkout(),
[
'set +x',
('export CODECOV_TOKEN=$(curl -sL http://taskcluster/secrets'
'/v1/secret/repo:github.com/glandium.git-cinnabar:codecov | '
'python -c "import json, sys; print(json.load(sys.stdin)'
'[\\"secret\\"][\\"token\\"])")'),
'set -x',
'cd repo',
],
upload_coverage,
)),
def __init__(self, **kwargs):
git = kwargs.pop('git', GIT_VERSION)
hg = kwargs.pop('hg', MERCURIAL_VERSION)
commit = kwargs.pop('commit', None)
task_env = kwargs.pop('task_env', 'linux')
variant = kwargs.pop('variant', None)
helper = kwargs.pop('helper', None)
clone = kwargs.pop('clone', TC_COMMIT)
desc = kwargs.pop('description', None)
extra_desc = kwargs.pop('extra_desc', None)
pre_command = kwargs.pop('pre_command', None)
if helper is None:
helper = '{}.{}'.format(task_env, variant) if variant else task_env
helper = Helper.install(helper)
if variant:
kwargs.setdefault('env', {})['VARIANT'] = variant
env = TaskEnvironment.by_name('{}.test'.format(task_env))
command = []
if pre_command:
command.extend(pre_command)
if hg:
command.extend(Hg.install('{}.{}'.format(task_env, hg)))
command.append('hg --version')
try:
if StrictVersion(hg) < '3.6':
kwargs.setdefault('env', {})['NO_CLONEBUNDLES'] = '1'
except ValueError:
# `hg` is a sha1 for trunk, which means it's >= 3.6
pass
if git:
command.extend(Git.install('{}.{}'.format(task_env, git)))
command.append('git --version')
command.extend(Task.checkout(commit=commit))
command.extend(helper)
if clone:
command.extend([
'curl -L {{{}.artifact}} -o repo/bundle.git'.format(
Clone.by_name(clone)),
'git init repo/hg.old.git',
'git -C repo/hg.old.git fetch ../bundle.git refs/*:refs/*',
'git -C repo/hg.old.git remote add origin hg::$REPO',
'git -C repo/hg.old.git symbolic-ref HEAD'
' refs/heads/branches/default/tip',
])
kwargs.setdefault('env', {})['REPO'] = REPO
if variant == 'coverage':
command = [
'export GIT_CINNABAR_COVERAGE=1',
'export COVERAGE_FILE=$PWD/repo/.coverage',
] + command
if 'command' in kwargs:
kwargs['command'] = command + kwargs['command']
else:
if commit:
# Always use the current CI scripts
command.append('git -C repo checkout {} CI'.format(TC_COMMIT))
kwargs['command'] = command + [
'make -C repo -f CI/tests.mk',
]
if variant == 'coverage':
kwargs['command'].extend([
'shopt -s nullglob',
'for f in repo/git-core/{{cinnabar,connect,hg}}*.gcda',
'do mv $f repo/helper',
'done',
'cd repo',
'zip $ARTIFACTS/coverage.zip .coverage'
' helper/{{cinnabar,connect,hg}}*.gcda',
'cd ..',
'shopt -u nullglob',
])
artifact = kwargs.pop('artifact', None)
artifacts = kwargs.setdefault('artifacts', [])
assert not (artifacts and artifact)
if artifact:
artifacts.push(artifact)
artifacts.append('coverage.zip')
self.coverage.append(self)
if not desc:
desc = 'test w/ git-{} hg-{}'.format(
git, 'r' + hg if len(hg) == 40 else hg)
if variant and variant != 'coverage':
desc = ' '.join((desc, variant))
if extra_desc:
desc = ' '.join((desc, extra_desc))
if task_env != 'linux':
desc = ' '.join((desc, env.os, env.cpu))
kwargs['description'] = desc
Task.__init__(self, task_env=env, **kwargs)
upload_coverage.extend([
'(' + '& '.join(download_coverage) + '& wait)',
'tar -Jxf gcda-helper.tar.xz',
])
for task in TestTask.coverage:
upload_coverage.extend([
'tar -Jxf cov-{{{}.id}}.tar.xz'.format(task),
'codecov --name "{}" --commit {} --branch {}'.format(
task.task['metadata']['name'], TC_COMMIT, TC_BRANCH),
('find . \\( -name .coverage -o -name coverage.xml '
'-o -name \\*.gcda -o -name \\*.gcov \\) -delete'),
])
if upload_coverage:
Task(
task_env=TaskEnvironment.by_name('linux.codecov'),
description='upload coverage',
scopes=['secrets:get:repo:github.com/glandium.git-cinnabar:codecov'],
command=list(chain(
Task.checkout(),
[
'set +x',
('export CODECOV_TOKEN=$(curl -sL http://taskcluster/secrets'
'/v1/secret/repo:github.com/glandium.git-cinnabar:codecov | '
'python -c "import json, sys; print(json.load(sys.stdin)'
'[\\"secret\\"][\\"token\\"])")'),
'set -x',
'cd repo',
],
upload_coverage,
)),