def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get('kind') == "system":
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
os.rename('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
os.rename('%s.tar.gz' % cachefile, cachefile)
unpack(defs, this, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url', 'http://foo.bar/') != 'http://foo.bar/':
if this.get('kind', 'chunk') == 'chunk':
with app.timer(this, 'upload'):
upload(defs, this)
def extract_commit(name, repo, ref, target_dir):
'''Check out a single commit (or tree) from a Git repo.
The checkout() function actually clones the entire repo, so this
function is much quicker when you don't need to copy the whole repo into
target_dir.
'''
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
with tempfile.NamedTemporaryFile() as git_index_file:
git_env = os.environ.copy()
git_env['GIT_INDEX_FILE'] = git_index_file.name
git_env['GIT_WORK_TREE'] = target_dir
app.log(name, 'Extracting commit', ref)
if call(['git', 'read-tree', ref], env=git_env, cwd=gitdir):
app.log(name, 'git read-tree failed for', ref, exit=True)
app.log(name, 'Then checkout index', ref)
if call(['git', 'checkout-index', '--all'], env=git_env, cwd=gitdir):
app.log(name, 'Git checkout-index failed for', ref, exit=True)
app.log(name, 'Done', ref)
utils.set_mtime_recursively(target_dir)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get('kind') == "system":
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
shutil.move('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
shutil.move('%s.tar.gz' % cachefile, cachefile)
app.config['counter'].increment()
unpack(defs, this, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url') is not None:
if this.get('kind', 'chunk') in ['chunk', 'stratum']:
with app.timer(this, 'upload'):
upload(defs, this)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config["tmp"]
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get("kind") == "system":
utils.hardlink_all_files(this["install"], this["sandbox"])
shutil.rmtree(this["install"])
shutil.rmtree(this["build"])
utils.set_mtime_recursively(this["sandbox"])
utils.make_deterministic_tar_archive(cachefile, this["sandbox"])
os.rename("%s.tar" % cachefile, cachefile)
else:
utils.set_mtime_recursively(this["install"])
utils.make_deterministic_gztar_archive(cachefile, this["install"])
os.rename("%s.tar.gz" % cachefile, cachefile)
unpack(defs, this, cachefile)
if (
app.config.get("kbas-password", "insecure") != "insecure"
and app.config.get("kbas-url", "http://foo.bar/") != "http://foo.bar/"
):
if this.get("kind") is not "cluster":
with app.timer(this, "upload"):
upload(defs, this)
def checkout(name, repo, ref, checkout):
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
# checkout the required version of this from git
with open(os.devnull, "w") as fnull:
# We need to pass '--no-hardlinks' because right now there's nothing to
# stop the build from overwriting the files in the .git directory
# inside the sandbox. If they were hardlinks, it'd be possible for a
# build to corrupt the repo cache. I think it would be faster if we
# removed --no-hardlinks, though.
if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
stdout=fnull, stderr=fnull):
app.exit(name, 'ERROR: git clone failed for', ref)
with app.chdir(checkout):
if call(['git', 'checkout', '--force', ref], stdout=fnull,
stderr=fnull):
app.exit(name, 'ERROR: git checkout failed for', ref)
app.log(name, 'Git checkout %s in %s' % (repo, checkout))
app.log(name, 'Upstream version %s' % get_version(checkout, ref))
if os.path.exists('.gitmodules'):
checkout_submodules(name, ref)
utils.set_mtime_recursively(checkout)
def extract_commit(name, repo, ref, target_dir):
'''Check out a single commit (or tree) from a Git repo.
The checkout() function actually clones the entire repo, so this
function is much quicker when you don't need to copy the whole repo into
target_dir.
'''
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
with tempfile.NamedTemporaryFile() as git_index_file:
git_env = os.environ.copy()
git_env['GIT_INDEX_FILE'] = git_index_file.name
git_env['GIT_WORK_TREE'] = target_dir
app.log(name, 'Extracting commit', ref)
if call(['git', 'read-tree', ref], env=git_env, cwd=gitdir):
app.exit(name, 'ERROR: git read-tree failed for', ref)
app.log(name, 'Then checkout index', ref)
if call(['git', 'checkout-index', '--all'], env=git_env, cwd=gitdir):
app.exit(name, 'ERROR: git checkout-index failed for', ref)
app.log(name, 'Done', ref)
utils.set_mtime_recursively(target_dir)
def checkout(name, repo, ref, checkout):
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
# checkout the required version of this from git
with open(os.devnull, "w") as fnull:
# We need to pass '--no-hardlinks' because right now there's nothing to
# stop the build from overwriting the files in the .git directory
# inside the sandbox. If they were hardlinks, it'd be possible for a
# build to corrupt the repo cache. I think it would be faster if we
# removed --no-hardlinks, though.
if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
stdout=fnull, stderr=fnull):
app.exit(name, 'ERROR: git clone failed for', ref)
with app.chdir(checkout):
if call(['git', 'checkout', '--force', ref], stdout=fnull,
stderr=fnull):
app.exit(name, 'ERROR: git checkout failed for', ref)
app.log(name, 'Git checkout %s in %s' % (repo, checkout))
app.log(name, 'Upstream version %s' % get_version(checkout, ref))
if os.path.exists('.gitmodules'):
checkout_submodules(name, ref)
utils.set_mtime_recursively(checkout)
def extract_commit(name, repo, ref, target_dir):
"""Check out a single commit (or tree) from a Git repo.
The checkout() function actually clones the entire repo, so this
function is much quicker when you don't need to copy the whole repo into
target_dir.
"""
gitdir = os.path.join(app.config["gits"], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
with tempfile.NamedTemporaryFile() as git_index_file:
git_env = os.environ.copy()
git_env["GIT_INDEX_FILE"] = git_index_file.name
git_env["GIT_WORK_TREE"] = target_dir
app.log(name, "Extracting commit", ref)
if call(["git", "read-tree", ref], env=git_env, cwd=gitdir):
app.exit(name, "ERROR: git read-tree failed for", ref)
app.log(name, "Then checkout index", ref)
if call(["git", "checkout-index", "--all"], env=git_env, cwd=gitdir):
app.exit(name, "ERROR: git checkout-index failed for", ref)
app.log(name, "Done", ref)
utils.set_mtime_recursively(target_dir)
def cache(dn):
if get_cache(dn):
app.log(dn, "Bah! I could have cached", cache_key(dn))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(dn))
if dn.get('kind') == "system":
utils.hardlink_all_files(dn['install'], dn['sandbox'])
shutil.rmtree(dn['checkout'])
utils.set_mtime_recursively(dn['install'])
utils.make_deterministic_tar_archive(cachefile, dn['install'])
shutil.move('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(dn['install'])
utils.make_deterministic_gztar_archive(cachefile, dn['install'])
shutil.move('%s.tar.gz' % cachefile, cachefile)
app.config['counter'].increment()
unpack(dn, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url') is not None:
if dn.get('kind', 'chunk') in app.config.get('kbas-upload', 'chunk'):
with app.timer(dn, 'upload'):
upload(dn)
def cache(defs, this, full_root=False):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if full_root:
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
os.rename('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
os.rename('%s.tar.gz' % cachefile, cachefile)
unpackdir = cachefile + '.unpacked'
os.makedirs(unpackdir)
if call(['tar', 'xf', cachefile, '--directory', unpackdir]):
app.exit(this, 'ERROR: Problem unpacking', cachefile)
try:
target = os.path.join(app.config['artifacts'], cache_key(defs, this))
os.rename(tmpdir, target)
size = os.path.getsize(get_cache(defs, this))
app.log(this, 'Now cached %s bytes as' % size, cache_key(defs, this))
except:
app.log(this, 'Bah! I raced and rebuilt', cache_key(defs, this))
def checkout(this):
_checkout(this['name'], this['repo'], this['ref'], this['build'])
with app.chdir(this['build']):
if os.path.exists('.gitmodules') or this.get('submodules'):
checkout_submodules(this)
utils.set_mtime_recursively(this['build'])
def checkout(dn):
_checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])
with app.chdir(dn['checkout']):
if os.path.exists('.gitmodules') or dn.get('submodules'):
checkout_submodules(dn)
utils.set_mtime_recursively(dn['checkout'])
def checkout(dn):
_checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])
with app.chdir(dn['checkout']):
if os.path.exists('.gitmodules') or dn.get('submodules'):
checkout_submodules(dn)
utils.set_mtime_recursively(dn['checkout'])
def checkout(this):
_checkout(this['name'], this['repo'], this['ref'], this['build'])
with app.chdir(this['build']):
if os.path.exists('.gitmodules') or this.get('submodules'):
checkout_submodules(this)
utils.set_mtime_recursively(this['build'])
def cache(defs, this, full_root=False):
app.log(this, "Creating cache artifact")
cachefile = os.path.join(app.settings['artifacts'], cache_key(defs, this))
if full_root:
shutil.make_archive(cachefile, 'tar', this['sandbox'])
os.rename('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
shutil.make_archive(cachefile, 'gztar', this['install'])
os.rename('%s.tar.gz' % cachefile, cachefile)
app.log(this, 'Now cached as', cache_key(defs, this))
if os.fork() == 0:
upload(this, cachefile)
sys.exit()
def checkout(name, repo, ref, checkoutdir):
gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
app.log(name, 'Upstream version:', get_upstream_version(repo, ref))
app.log(name, 'Git checkout %s in %s' % (repo, checkoutdir))
# checkout the required version of this from git
with app.chdir(checkoutdir), open(os.devnull, "w") as fnull:
copy_repo(gitdir, checkoutdir)
if call(['git', 'checkout', ref], stdout=fnull, stderr=fnull):
app.log(name, 'ERROR: git checkout failed for', ref)
raise SystemExit
if os.path.exists('.gitmodules'):
checkout_submodules(name, ref)
utils.set_mtime_recursively(checkoutdir)
def cache(this):
cachefile = os.path.join(app.settings['artifacts'], cache_key(this))
utils.set_mtime_recursively(this['install'])
shutil.make_archive(cachefile, 'gztar', this['install'])
app.log(this, 'Now cached as', cache_key(this))