def install_helper(name):
helper = Helper.by_name(name)
filename = os.path.basename(helper.artifacts[0])
return [
'curl --compressed -o {} -L {{{}.artifacts[0]}}'.format(
filename, Helper.by_name(name)),
'chmod +x {}'.format(filename),
'git config --global cinnabar.helper $PWD/{}'.format(filename),
]
def generate_images_post():
if "image" not in request.form:
return "image must be provided", status.HTTP_400_BAD_REQUEST
base64Img = request.form['image']
print("POST base64Img :", type(base64Img))
print("POST base64Img :", base64Img)
base64Img = base64Img.replace(" ", "+")
ext = guess_extension(guess_type(base64Img)[0])
with tempfile.TemporaryDirectory() as folder:
local_full_path_file = os.path.join(
folder, "{0}{1}".format(Helper.generate_name(), ext))
exts = ["jpeg", "jpg", "png", "gif", "tiff"]
with open(local_full_path_file, 'wb') as f:
content = base64Img.split(',')[1]
encoded_content = content.encode()
decoded = Helper.decode_base64(encoded_content)
f.write(decoded)
_, hed = edge_transformer.transform(local_full_path_file)
filename_hed = os.path.join(
folder, "{0}_hed_{1}".format(Helper.generate_name(), ext))
cv2.imwrite(filename_hed, hed)
with open(filename_hed, "rb") as image_file:
encoded_string = base64.b64encode(
image_file.read()).decode('ascii')
hed_to_return = np.invert(hed) #inverse color
filename_hed_to_return = os.path.join(
folder, "{0}_hed_to_return_{1}".format(Helper.generate_name(),
ext))
cv2.imwrite(filename_hed_to_return, hed_to_return)
with open(filename_hed_to_return, "rb") as image_file:
hed_to_return_encoded_string = base64.b64encode(
image_file.read()).decode('ascii')
hed_image_base64 = "data:image/png;base64," + hed_to_return_encoded_string
mask = Image.open(filename_hed)
mask = transforms(mask)
mask = mask.unsqueeze(0)
mask = nn.functional.interpolate(mask, size=image_shape)
mask = mask.to(device)
with torch.no_grad():
generated_image = gen(mask)
generated_image_file = os.path.join(folder,
Helper.generate_name() + ".png")
save_image(generated_image[0], generated_image_file)
with open(generated_image_file, "rb") as image_file:
encoded_string = base64.b64encode(
image_file.read()).decode('ascii')
generated_image_base64 = "data:image/png;base64," + encoded_string
return jsonify({
"hed": hed_image_base64,
"generated": generated_image_base64
})
def __init__(self, version):
sha1 = git_rev_parse(version)
expireIn = '26 weeks'
if version == TC_COMMIT or len(version) == 40:
if version == TC_COMMIT:
download = Helper.install('linux')
else:
download = Helper.install('linux.old:{}'.format(version))
expireIn = '26 weeks'
elif parse_version(version) > parse_version('0.5.0a'):
download = ['repo/git-cinnabar download']
elif parse_version(version) == parse_version('0.4.0'):
download = ['(cd repo ; ./git-cinnabar download)']
else:
download = []
if (parse_version(version) < parse_version('0.5.0b3')
and version != TC_COMMIT):
hg = '4.3.3'
else:
hg = MERCURIAL_VERSION
if REPO == DEFAULT_REPO:
index = 'bundle.{}'.format(sha1)
else:
index = 'bundle.{}.{}'.format(hashlib.sha1(REPO).hexdigest(), sha1)
TestTask.__init__(
self,
hg=hg,
description='clone w/ {}'.format(version),
index=index,
expireIn=expireIn,
helper=download,
commit=sha1,
clone=False,
command=[
'PATH=$PWD/repo:$PATH'
' git -c fetch.prune=true clone -n hg::$REPO hg.old.git',
'git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all',
],
artifact='bundle.git',
env={
'REPO': REPO,
},
priority='high',
)
def __init__(self, version):
sha1 = git_rev_parse(version)
expireIn = '26 weeks'
if version == TC_COMMIT or len(version) == 40:
if version == TC_COMMIT:
download = Helper.install('linux')
else:
download = Helper.install('linux.old:{}'.format(version))
expireIn = '26 weeks'
elif parse_version(version) > parse_version('0.5.0a'):
download = ['repo/git-cinnabar download']
elif parse_version(version) == parse_version('0.4.0'):
download = ['(cd repo ; ./git-cinnabar download)']
else:
download = []
if (parse_version(version) < parse_version('0.5.0b3') and
version != TC_COMMIT):
hg = '4.3.3'
else:
hg = MERCURIAL_VERSION
if REPO == DEFAULT_REPO:
index = 'bundle.{}'.format(sha1)
else:
index = 'bundle.{}.{}'.format(hashlib.sha1(REPO).hexdigest(), sha1)
TestTask.__init__(
self,
hg=hg,
description='clone w/ {}'.format(version),
index=index,
expireIn=expireIn,
helper=download,
commit=sha1,
clone=False,
command=[
'PATH=$PWD/repo:$PATH'
' git -c fetch.prune=true clone -n hg::$REPO hg.old.git',
'git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all',
],
artifact='bundle.git',
env={
'REPO': REPO,
},
)
def predict_car_type():
if "images" not in request.form:
return "images must be provided", status.HTTP_400_BAD_REQUEST
images = json.loads(request.form['images'])
images_to_predict = Helper.get_fixed_base64_images_by_dict(images)
model = model_loader.load(
os.path.join(os.getcwd(), "models/Resnet50_with_traning_best.hdf5"))
predictions = predictor.predict(model, images_to_predict, (224, 224),
["car", "jeep", "truck"])
# the order {'car': 0, 'jeep': 1, 'truck': 2} come from train_generator.class_indices, validation_generator.class_indices or test_generator.class_indices
# the 3 give some order for classes
return jsonify(predictions)
def generate_shoe():
base64Img = request.args.get('image')
base64Img = base64Img.replace(" ", "+")
base64Img = Helper.get_fixed_base64_image(base64Img)
decoded_img = base64.b64decode(base64Img)
img_buffer = BytesIO(decoded_img)
imageData = Image.open(img_buffer).convert('LA')
#num_channel = len(imageData.split())
#print("num_channel:", num_channel)
img = ImageOps.fit(imageData, image_shape)
img_tensor = transforms(img)
img_tensor = img_tensor.unsqueeze(0)
with torch.no_grad():
generated_image = gen(img_tensor[:, 1:2, :, :])
print("generated_image.shape : ", generated_image.shape)
save_image(generated_image[0], "image1.png")
with open("image1.png", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode('ascii')
return jsonify("data:image/png;base64," + encoded_string)
def generate_shoe_by_hed():
base64Img = request.args.get('image')
base64Img = base64Img.replace(" ", "+")
""""""
base64Img = Helper.get_fixed_base64_image(base64Img)
decoded_img = base64.b64decode(base64Img)
img_buffer = BytesIO(decoded_img)
imageData = Image.open(img_buffer).convert('LA')
img = ImageOps.fit(imageData, image_shape)
img_tensor = transforms(img)
img_tensor = img_tensor.unsqueeze(0)
print("img_tensor.shape : ", img_tensor.shape)
img = cv2.imdecode(img, cv2.IMREAD_COLOR)
cv2.imshow(img)
cv2.waitKey(0)
with torch.no_grad():
generated_image = gen(img_tensor[:, 0:1, :, :])
print("generated_image.shape : ", generated_image.shape)
save_image(generated_image[0], "image1.png")
with open("image1.png", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode('ascii')
return jsonify("data:image/png;base64," + encoded_string)
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests)',
'(cd repo && flake8 --ignore E402 $(git ls-files \*\*.py'
' git-cinnabar git-remote-hg))',
],
)
for env in ('linux', 'mingw64', 'osx10_10'):
TestTask(task_env=env)
for env in ('linux', 'mingw64', 'osx10_11'):
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
for env in ('linux', 'osx10_11'):
TestTask(
task_env=env,
variant='asan',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='coverage',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
def __init__(self, **kwargs):
git = kwargs.pop('git', GIT_VERSION)
hg = kwargs.pop('hg', MERCURIAL_VERSION)
commit = kwargs.pop('commit', None)
task_env = kwargs.pop('task_env', 'linux')
variant = kwargs.pop('variant', None)
helper = kwargs.pop('helper', None)
clone = kwargs.pop('clone', TC_COMMIT)
desc = kwargs.pop('description', None)
extra_desc = kwargs.pop('extra_desc', None)
pre_command = kwargs.pop('pre_command', None)
if helper is None:
helper = '{}.{}'.format(task_env, variant) if variant else task_env
helper = Helper.install(helper)
if variant:
kwargs.setdefault('env', {})['VARIANT'] = variant
env = TaskEnvironment.by_name('{}.test'.format(task_env))
command = []
if pre_command:
command.extend(pre_command)
if hg:
command.extend(Hg.install('{}.{}'.format(task_env, hg)))
command.append('hg --version')
try:
if StrictVersion(hg) < '3.6':
kwargs.setdefault('env', {})['NO_CLONEBUNDLES'] = '1'
except ValueError:
# `hg` is a sha1 for trunk, which means it's >= 3.6
pass
if git:
command.extend(Git.install('{}.{}'.format(task_env, git)))
command.append('git --version')
command.extend(Task.checkout(commit=commit))
command.extend(helper)
if clone:
command.extend([
'curl -L {{{}.artifact}} -o repo/bundle.git'.format(
Clone.by_name(clone)),
'git init repo/hg.old.git',
'git -C repo/hg.old.git fetch ../bundle.git refs/*:refs/*',
'git -C repo/hg.old.git remote add origin hg::$REPO',
'git -C repo/hg.old.git symbolic-ref HEAD'
' refs/heads/branches/default/tip',
])
kwargs.setdefault('env', {})['REPO'] = REPO
if variant == 'coverage':
command = [
'export GIT_CINNABAR_COVERAGE=1',
'export COVERAGE_FILE=$PWD/repo/.coverage',
] + command
if 'command' in kwargs:
kwargs['command'] = command + kwargs['command']
else:
if commit:
# Always use the current CI scripts
command.append('git -C repo checkout {} CI'.format(TC_COMMIT))
kwargs['command'] = command + [
'make -C repo -f CI/tests.mk',
]
if variant == 'coverage':
kwargs['command'].extend([
'shopt -s nullglob',
'for f in repo/git-core/{{cinnabar,connect,hg}}*.gcda',
'do mv $f repo/helper',
'done',
'cd repo',
'zip $ARTIFACTS/coverage.zip .coverage'
' helper/{{cinnabar,connect,hg}}*.gcda',
'cd ..',
'shopt -u nullglob',
])
artifact = kwargs.pop('artifact', None)
artifacts = kwargs.setdefault('artifacts', [])
assert not (artifacts and artifact)
if artifact:
artifacts.push(artifact)
artifacts.append('coverage.zip')
self.coverage.append(self)
if not desc:
desc = 'test w/ git-{} hg-{}'.format(
git, 'r' + hg if len(hg) == 40 else hg)
if variant and variant != 'coverage':
desc = ' '.join((desc, variant))
if extra_desc:
desc = ' '.join((desc, extra_desc))
if task_env != 'linux':
desc = ' '.join((desc, env.os, env.cpu))
kwargs['description'] = desc
Task.__init__(self, task_env=env, **kwargs)
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests)',
'(cd repo && flake8 --ignore E402 $(git ls-files \\*\\*.py'
' git-cinnabar git-remote-hg | grep -v ^CI/))',
'(cd repo && python3 -m flake8 --ignore E402 '
'$(git ls-files CI/\\*\\*.py))',
],
)
TestTask(
description='cram tests',
variant='coverage',
clone=False,
command=[
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK': 'no-version-check',
},
)
for env in ('linux', 'mingw64', 'osx10_10'):
TestTask(task_env=env)
for env in ('linux', 'mingw64', 'osx10_11'):
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
for env in ('linux', 'osx10_11'):
TestTask(
task_env=env,
variant='asan',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
env={
'GIT_CINNABAR_OLD': '1',
},
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GIT_CINNABAR_OLD': '1',
'GRAFT': '1',
},
)
TestTask(
variant='coverage',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
for variant in ('coverage', 'old'):
env = {
'GIT_CINNABAR_CHECK': 'no-mercurial',
}
if variant == 'old':
env['GIT_CINNABAR_OLD_HELPER'] = '1'
TestTask(
variant=variant,
extra_desc='no-mercurial',
pre_command=[
'python -m virtualenv venv',
'. venv/bin/activate',
],
command=[
'deactivate',
# deactivate removes the git directory from $PATH.
# Also add the virtualenv bin directory to $PATH for mercurial
# to be found, but at the end for the system python to still
# be picked.
'export PATH=$PWD/git/bin:$PATH:$PWD/venv/bin',
'make -C repo -f CI/tests.mk',
],
env=env
)
def decision():
TestTask(
description='python lint & tests',
variant='coverage',
clone=False,
command=[
'(cd repo &&'
' nosetests --all-modules --with-coverage --cover-tests tests &&'
' nosetests3 --all-modules tests)',
'(cd repo && python -m flake8 --ignore E402,F405'
' $(git ls-files \\*\\*.py git-cinnabar git-remote-hg'
' | grep -v ^CI/))',
'(cd repo && flake8 --ignore E402,F405'
' $(git ls-files CI/\\*\\*.py)'
' $(git grep -l unicode_literals))',
],
)
for env in ('linux', 'mingw64', 'osx10_10'):
# Can't spawn osx workers from pull requests.
if env.startswith('osx') and not TC_IS_PUSH:
continue
TestTask(task_env=env)
task_env = TaskEnvironment.by_name('{}.test'.format(env))
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os,
task_env.cpu),
command=list(
chain(
Git.install('{}.{}'.format(env, GIT_VERSION)),
Hg.install('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
[
'(cd repo ; ./git-cinnabar download --dev)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download --dev)',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
} if env == 'linux' else {},
)
# Because nothing is using the x86 windows helper, we need to manually
# touch it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
)
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
clone=upgrade,
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
'GIT_CINNABAR_LOG': 'reexec:3',
'UPGRADE_FROM': upgrade,
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
for git in ('1.8.5', '2.7.4'):
TestTask(git=git)
for hg in SOME_MERCURIAL_VERSIONS:
if hg != MERCURIAL_VERSION:
TestTask(hg=hg)
TestTask(
task_env='linux',
variant='asan',
)
TestTask(
task_env='linux',
variant='asan',
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='graft',
env={
'GRAFT': '1',
},
)
TestTask(
variant='old',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
},
)
TestTask(
variant='old',
extra_desc='graft',
env={
'GIT_CINNABAR_OLD_HELPER': '1',
'GRAFT': '1',
},
)
rev = old_compatible_python()
TestTask(
commit=rev,
clone=rev,
extra_desc='old python',
env={
'GIT_CINNABAR_OLD': '1',
},
)
TestTask(
commit=rev,
clone=rev,
extra_desc='old python graft',
env={
'GIT_CINNABAR_OLD': '1',
'GRAFT': '1',
},
)
TestTask(
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3',
'GRAFT': '1',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
},
)
TestTask(
variant='coverage',
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'true',
'GRAFT': '1',
},
)
TestTask(
extra_desc='experiments',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
'GIT_CINNABAR_LOG': 'reexec:3',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
TestTask(
extra_desc='experiments graft',
env={
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
'GIT_CINNABAR_LOG': 'reexec:3',
'GRAFT': '1',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
for variant in ('coverage', 'old'):
env = {
'GIT_CINNABAR_CHECK': 'no-mercurial',
}
if variant == 'old':
env['GIT_CINNABAR_OLD_HELPER'] = '1'
TestTask(
variant=variant,
extra_desc='no-mercurial',
pre_command=[
'python -m virtualenv venv',
'. venv/bin/activate',
],
command=[
'deactivate',
# deactivate removes the git directory from $PATH.
# Also add the virtualenv bin directory to $PATH for mercurial
# to be found, but at the end for the system python to still
# be picked.
'export PATH=$PWD/git/bin:$PATH:$PWD/venv/bin',
'make -C repo -f CI/tests.mk',
],
env=env)
for variant in ('coverage', 'asan'):
for check in ([], ['no-mercurial']):
TestTask(
variant=variant,
extra_desc=' '.join(['cram'] + check),
clone=False,
command=[
'repo/git-cinnabar --version',
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK':
','.join(['no-version-check'] + check),
},
)
for check in ([], ['no-mercurial']):
TestTask(
extra_desc=' '.join(['cram'] + check),
clone=False,
command=[
'repo/git-cinnabar --version',
'cram --verbose repo/tests',
],
env={
'GIT_CINNABAR_CHECK': ','.join(['no-version-check'] + check),
'GIT_CINNABAR_EXPERIMENTS': 'python3,true',
},
hg='{}.py3'.format(MERCURIAL_VERSION),
)
def main():
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
merge_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.zip -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcno-helper.zip -L {{{}.artifacts[1]}}'.format(task))
merge_coverage.append('(' + '& '.join(download_coverage) + '& wait)', )
for task in TestTask.coverage:
merge_coverage.extend([
'unzip -d cov-{{{}.id}} cov-{{{}.id}}.zip .coverage'.format(
task, task),
])
merge_coverage.extend([
'grcov -s repo -t lcov -o repo/coverage.lcov gcno-helper.zip ' +
' '.join('cov-{{{}.id}}.zip'.format(task)
for task in TestTask.coverage),
'cd repo',
'coverage combine --append {}'.format(' '.join(
'../cov-{{{}.id}}/.coverage'.format(task)
for task in TestTask.coverage)),
'cd ..',
])
if merge_coverage:
Task(
task_env=TaskEnvironment.by_name('linux.codecov'),
description='upload coverage',
scopes=['secrets:get:project/git-cinnabar/codecov'],
command=list(
chain(
Task.checkout(),
[
'set +x',
('export CODECOV_TOKEN=$(curl -sL '
'http://taskcluster/api/secrets/v1/secret/project/git-'
'cinnabar/codecov | '
'python -c "import json, sys; print(json.load(sys.stdin)'
'[\\"secret\\"][\\"token\\"])")'),
'set -x',
],
merge_coverage,
[
'cd repo',
'codecov --name "taskcluster" --commit {} --branch {}'.
format(TC_COMMIT, TC_BRANCH),
],
)),
)
for t in Task.by_id.values():
t.submit()
if not TC_ACTION and 'TC_GROUP_ID' in os.environ:
actions = {
'version': 1,
'actions': [],
'variables': {
'e': dict(TC_DATA, decision_id=''),
'tasks_for': 'action',
},
}
for name, a in action.by_name.items():
if name != 'decision':
actions['actions'].append({
'kind': 'task',
'name': a.name,
'title': a.title,
'description': a.description,
'context': [],
'task': a.task,
})
with open('actions.json', 'w') as out:
out.write(json.dumps(actions, indent=True))
def __init__(self, **kwargs):
git = kwargs.pop('git', GIT_VERSION)
hg = kwargs.pop('hg', MERCURIAL_VERSION)
commit = kwargs.pop('commit', None)
task_env = kwargs.pop('task_env', 'linux')
variant = kwargs.pop('variant', None)
helper = kwargs.pop('helper', None)
clone = kwargs.pop('clone', TC_COMMIT)
desc = kwargs.pop('description', None)
extra_desc = kwargs.pop('extra_desc', None)
pre_command = kwargs.pop('pre_command', None)
if helper is None:
helper = '{}.{}'.format(task_env, variant) if variant else task_env
helper = Helper.install(helper)
if variant:
kwargs.setdefault('env', {})['VARIANT'] = variant
env = TaskEnvironment.by_name('{}.test'.format(task_env))
command = []
if pre_command:
command.extend(pre_command)
if hg:
command.extend(Hg.install('{}.{}'.format(task_env, hg)))
command.append('hg --version')
if LooseVersion(hg) < '3.6':
kwargs.setdefault('env', {})['NO_CLONEBUNDLES'] = '1'
if git:
command.extend(Git.install('{}.{}'.format(task_env, git)))
command.append('git --version')
command.extend(Task.checkout(commit=commit))
command.extend(helper)
if clone:
command.extend([
'curl -L {{{}.artifact}} -o repo/bundle.git'.format(
Clone.by_name(clone)),
'git init repo/hg.old.git',
'git -C repo/hg.old.git fetch ../bundle.git refs/*:refs/*',
'git -C repo/hg.old.git remote add origin hg::$REPO',
'git -C repo/hg.old.git symbolic-ref HEAD'
' refs/heads/branches/default/tip',
])
kwargs.setdefault('env', {})['REPO'] = REPO
if 'command' in kwargs:
kwargs['command'] = command + kwargs['command']
else:
if commit:
# Always use the current CI scripts
command.append('git -C repo checkout {} CI'.format(TC_COMMIT))
kwargs['command'] = command + [
'make -C repo -f CI/tests.mk',
]
if variant == 'coverage':
kwargs['command'].extend([
'shopt -s nullglob',
'for f in repo/git-core/{{cinnabar,connect,hg}}*.gcda',
'do mv $f repo/helper',
'done',
'cd repo',
'tar -Jcf $ARTIFACTS/coverage.tar.xz .coverage'
' helper/{{cinnabar,connect,hg}}*.gcda',
'cd ..',
'shopt -u nullglob',
])
artifact = kwargs.pop('artifact', None)
artifacts = kwargs.setdefault('artifacts', [])
assert not(artifacts and artifact)
if artifact:
artifacts.push(artifact)
artifacts.append('coverage.tar.xz')
self.coverage.append(self)
if not desc:
desc = 'test w/ git-{} hg-{}'.format(
git, 'r' + hg if len(hg) == 40 else hg)
if variant and variant != 'coverage':
desc = ' '.join((desc, variant))
if extra_desc:
desc = ' '.join((desc, extra_desc))
if task_env != 'linux':
desc = ' '.join((desc, env.os, env.cpu))
kwargs['description'] = desc
Task.__init__(self, task_env=env, **kwargs)
Task(
task_env=task_env,
description='download helper {} {}'.format(task_env.os, task_env.cpu),
command=list(
chain(
install_git('{}.{}'.format(env, GIT_VERSION)),
install_hg('{}.{}'.format(env, MERCURIAL_VERSION)),
Task.checkout(),
requests + [
'(cd repo ; ./git-cinnabar download)',
'rm -rf repo/.git',
'(cd repo ; ./git-cinnabar download)',
],
)),
dependencies=[
Helper.by_name(env),
],
)
# Because nothing is using the i686 windows helper, we need to manually touch
# it.
Helper.by_name('mingw32')
for upgrade in UPGRADE_FROM:
TestTask(
extra_desc='upgrade-from-{}'.format(upgrade),
variant='coverage',
clone=upgrade,
env={
'UPGRADE_FROM': upgrade,
},
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
upload_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.tar.xz -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcda-helper.tar.xz -L {{{}.artifacts[1]}}'.format(task))
upload_coverage.extend([
'(' + '& '.join(download_coverage) + '& wait)',
'tar -Jxf gcda-helper.tar.xz',
])
for task in TestTask.coverage:
upload_coverage.extend([
'tar -Jxf cov-{{{}.id}}.tar.xz'.format(task),
'codecov --name "{}" --commit {} --branch {}'.format(
task.task['metadata']['name'], TC_COMMIT, TC_BRANCH),
('find . \\( -name .coverage -o -name coverage.xml '
'-o -name \\*.gcda -o -name \\*.gcov \\) -delete'),
])
try:
func = action.by_name[TC_ACTION or 'decision'].func
except AttributeError:
raise Exception('Unsupported action: %s', TC_ACTION or 'decision')
func()
upload_coverage = []
if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
download_coverage = [
'curl -o cov-{{{}.id}}.tar.xz -L {{{}.artifact}}'.format(task, task)
for task in TestTask.coverage
]
task = Helper.by_name('linux.coverage')
download_coverage.append(
'curl -o gcda-helper.tar.xz -L {{{}.artifacts[1]}}'.format(task))
upload_coverage.extend([
'(' + '& '.join(download_coverage) + '& wait)',
'tar -Jxf gcda-helper.tar.xz',
])
for task in TestTask.coverage:
upload_coverage.extend([
'tar -Jxf cov-{{{}.id}}.tar.xz'.format(task),
'codecov --name "{}" --commit {} --branch {}'.format(
task.task['metadata']['name'], TC_COMMIT, TC_BRANCH),
('find . \( -name .coverage -o -name coverage.xml -o -name \*.gcda'
' -o -name \*.gcov \) -delete'),
])
def __init__(self, **kwargs):
git = kwargs.pop('git', GIT_VERSION)
hg = kwargs.pop('hg', MERCURIAL_VERSION)
commit = kwargs.pop('commit', None)
task_env = kwargs.pop('task_env', 'linux')
variant = kwargs.pop('variant', None)
helper = kwargs.pop('helper', None)
clone = kwargs.pop('clone', TC_COMMIT)
desc = kwargs.pop('description', None)
extra_desc = kwargs.pop('extra_desc', None)
if helper is None:
helper = '{}.{}'.format(task_env, variant) if variant else task_env
helper = Helper.install(helper)
if variant:
kwargs.setdefault('env', {})['VARIANT'] = variant
env = TaskEnvironment.by_name('{}.test'.format(task_env))
command = []
if hg:
command.extend(Hg.install('{}.{}'.format(task_env, hg)))
command.append('hg --version')
if git:
command.extend(Git.install('{}.{}'.format(task_env, git)))
command.append('git --version')
command.extend(Task.checkout(commit=commit))
command.extend(helper)
if clone:
command.extend([
'curl -L {{{}.artifact}} -o clone.tar.xz'.format(
Clone.by_name(clone)),
'tar -C repo -Jxf clone.tar.xz',
])
if 'command' in kwargs:
kwargs['command'] = command + kwargs['command']
else:
if commit:
# Always use the current CI scripts
command.append('git -C repo checkout {} CI'.format(TC_COMMIT))
kwargs['command'] = command + [
'make -C repo -f CI/tests.mk',
]
if variant == 'coverage':
kwargs['command'].extend([
'shopt -s nullglob',
'for f in repo/git-core/{{cinnabar,connect,hg}}*.gcda',
'do mv $f repo/helper',
'done',
'cd repo',
'tar -Jcf $ARTIFACTS/coverage.tar.xz .coverage'
' helper/{{cinnabar,connect,hg}}*.gcda',
'cd ..',
'shopt -u nullglob',
])
artifact = kwargs.pop('artifact', None)
artifacts = kwargs.setdefault('artifacts', [])
assert not (artifacts and artifact)
if artifact:
artifacts.push(artifact)
artifacts.append('coverage.tar.xz')
self.coverage.append(self)
if not desc:
desc = 'test w/ git-{} hg-{}'.format(
git, 'r' + hg if len(hg) == 40 else hg)
if variant and variant != 'coverage':
desc = ' '.join((desc, variant))
if extra_desc:
desc = ' '.join((desc, extra_desc))
if task_env != 'linux':
desc = ' '.join((desc, env.os, env.cpu))
kwargs['description'] = desc
Task.__init__(self, task_env=env, **kwargs)