def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config["tmp"]
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get("kind") == "system":
utils.hardlink_all_files(this["install"], this["sandbox"])
shutil.rmtree(this["install"])
shutil.rmtree(this["build"])
utils.set_mtime_recursively(this["sandbox"])
utils.make_deterministic_tar_archive(cachefile, this["sandbox"])
os.rename("%s.tar" % cachefile, cachefile)
else:
utils.set_mtime_recursively(this["install"])
utils.make_deterministic_gztar_archive(cachefile, this["install"])
os.rename("%s.tar.gz" % cachefile, cachefile)
unpack(defs, this, cachefile)
if (
app.config.get("kbas-password", "insecure") != "insecure"
and app.config.get("kbas-url", "http://foo.bar/") != "http://foo.bar/"
):
if this.get("kind") is not "cluster":
with app.timer(this, "upload"):
upload(defs, this)
def cache(dn):
if get_cache(dn):
app.log(dn, "Bah! I could have cached", cache_key(dn))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(dn))
if dn.get('kind') == "system":
utils.hardlink_all_files(dn['install'], dn['sandbox'])
shutil.rmtree(dn['checkout'])
utils.set_mtime_recursively(dn['install'])
utils.make_deterministic_tar_archive(cachefile, dn['install'])
shutil.move('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(dn['install'])
utils.make_deterministic_gztar_archive(cachefile, dn['install'])
shutil.move('%s.tar.gz' % cachefile, cachefile)
app.config['counter'].increment()
unpack(dn, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url') is not None:
if dn.get('kind', 'chunk') in app.config.get('kbas-upload', 'chunk'):
with app.timer(dn, 'upload'):
upload(dn)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get('kind') == "system":
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
shutil.move('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
shutil.move('%s.tar.gz' % cachefile, cachefile)
app.config['counter'].increment()
unpack(defs, this, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url') is not None:
if this.get('kind', 'chunk') in ['chunk', 'stratum']:
with app.timer(this, 'upload'):
upload(defs, this)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get('kind') == "system":
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
os.rename('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
os.rename('%s.tar.gz' % cachefile, cachefile)
unpack(defs, this, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url', 'http://foo.bar/') != 'http://foo.bar/':
if this.get('kind', 'chunk') == 'chunk':
with app.timer(this, 'upload'):
upload(defs, this)
def cache(defs, this, full_root=False):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if full_root:
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
os.rename('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
os.rename('%s.tar.gz' % cachefile, cachefile)
unpackdir = cachefile + '.unpacked'
os.makedirs(unpackdir)
if call(['tar', 'xf', cachefile, '--directory', unpackdir]):
app.exit(this, 'ERROR: Problem unpacking', cachefile)
try:
target = os.path.join(app.config['artifacts'], cache_key(defs, this))
os.rename(tmpdir, target)
size = os.path.getsize(get_cache(defs, this))
app.log(this, 'Now cached %s bytes as' % size, cache_key(defs, this))
except:
app.log(this, 'Bah! I raced and rebuilt', cache_key(defs, this))
def install_artifact(this, component, permit_bootstrap=True):
component = Definitions().get(component)
if component.get('build-mode') == 'bootstrap' and permit_bootstrap == False:
return
app.log(this, 'Installing %s' % component['cache'])
unpackdir = cache.unpack(component)
utils.hardlink_all_files(unpackdir, this['assembly'])
def install(defs, this, component):
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
if app.config.get('log-verbose'):
app.log(this, 'Installing %s' % component['cache'])
unpackdir = cache.get_cache(defs, component) + '.unpacked'
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def install(defs, this, component):
# populate this['sandbox'] with the artifact files from component
if os.path.exists(os.path.join(this["sandbox"], "baserock", component["name"] + ".meta")):
return
if app.config.get("log-verbose"):
app.log(this, "Installing %s" % component["cache"])
if cache.get_cache(defs, component) is False:
app.exit(this, "ERROR: unable to get cache for", component["name"])
unpackdir = cache.get_cache(defs, component) + ".unpacked"
if this.get("kind") is "system":
utils.copy_all_files(unpackdir, this["sandbox"])
else:
utils.hardlink_all_files(unpackdir, this["sandbox"])
def install(dn, component):
# populate dn['sandbox'] with the artifact files from component
if os.path.exists(os.path.join(dn['sandbox'], 'baserock',
component['name'] + '.meta')):
return
app.log(dn, 'Sandbox: installing %s' % component['cache'], verbose=True)
if cache.get_cache(component) is False:
app.log(dn, 'Unable to get cache for', component['name'], exit=True)
unpackdir = cache.get_cache(component) + '.unpacked'
if dn.get('kind') is 'system':
utils.copy_all_files(unpackdir, dn['sandbox'])
else:
utils.hardlink_all_files(unpackdir, dn['sandbox'])
def install(defs, this, component):
# populate this['sandbox'] with the artifact files from component
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
if app.config.get('log-verbose'):
app.log(this, 'Sandbox: installing %s' % component['cache'])
if cache.get_cache(defs, component) is False:
app.exit(this, 'ERROR: unable to get cache for', component['name'])
unpackdir = cache.get_cache(defs, component) + '.unpacked'
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def install(dn, component):
# populate dn['sandbox'] with the artifact files from component
if os.path.exists(
os.path.join(dn['sandbox'], 'baserock',
component['name'] + '.meta')):
return
app.log(dn, 'Sandbox: installing %s' % component['cache'], verbose=True)
if cache.get_cache(component) is False:
app.log(dn, 'Unable to get cache for', component['name'], exit=True)
unpackdir = cache.get_cache(component) + '.unpacked'
if dn.get('kind') is 'system':
utils.copy_all_files(unpackdir, dn['sandbox'])
else:
utils.hardlink_all_files(unpackdir, dn['sandbox'])
def install(defs, this, component):
# populate this['sandbox'] with the artifact files from component
if os.path.exists(
os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
if app.config.get('log-verbose'):
app.log(this, 'Installing %s' % component['cache'])
if cache.get_cache(defs, component) is False:
app.exit(this, 'ERROR: unable to get cache for', component['name'])
unpackdir = cache.get_cache(defs, component) + '.unpacked'
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def _install(defs, this, component):
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
for it in component.get('build-depends', []):
dependency = defs.get(it)
if (dependency.get('build-mode', 'staging') ==
component.get('build-mode', 'staging')):
_install(defs, this, dependency)
for it in component.get('contents', []):
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
_install(defs, this, subcomponent)
unpackdir = cache.unpack(defs, component)
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
