def cache(defs, this): if get_cache(defs, this): app.log(this, "Bah! I could have cached", cache_key(defs, this)) return tempfile.tempdir = app.config["tmp"] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(defs, this)) if this.get("kind") == "system": utils.hardlink_all_files(this["install"], this["sandbox"]) shutil.rmtree(this["install"]) shutil.rmtree(this["build"]) utils.set_mtime_recursively(this["sandbox"]) utils.make_deterministic_tar_archive(cachefile, this["sandbox"]) os.rename("%s.tar" % cachefile, cachefile) else: utils.set_mtime_recursively(this["install"]) utils.make_deterministic_gztar_archive(cachefile, this["install"]) os.rename("%s.tar.gz" % cachefile, cachefile) unpack(defs, this, cachefile) if ( app.config.get("kbas-password", "insecure") != "insecure" and app.config.get("kbas-url", "http://foo.bar/") != "http://foo.bar/" ): if this.get("kind") is not "cluster": with app.timer(this, "upload"): upload(defs, this)
def cache(dn): if get_cache(dn): app.log(dn, "Bah! I could have cached", cache_key(dn)) return tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(dn)) if dn.get('kind') == "system": utils.hardlink_all_files(dn['install'], dn['sandbox']) shutil.rmtree(dn['checkout']) utils.set_mtime_recursively(dn['install']) utils.make_deterministic_tar_archive(cachefile, dn['install']) shutil.move('%s.tar' % cachefile, cachefile) else: utils.set_mtime_recursively(dn['install']) utils.make_deterministic_gztar_archive(cachefile, dn['install']) shutil.move('%s.tar.gz' % cachefile, cachefile) app.config['counter'].increment() unpack(dn, cachefile) if app.config.get('kbas-password', 'insecure') != 'insecure' and \ app.config.get('kbas-url') is not None: if dn.get('kind', 'chunk') in app.config.get('kbas-upload', 'chunk'): with app.timer(dn, 'upload'): upload(dn)
def cache(defs, this): if get_cache(defs, this): app.log(this, "Bah! I could have cached", cache_key(defs, this)) return tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(defs, this)) if this.get('kind') == "system": utils.hardlink_all_files(this['install'], this['sandbox']) shutil.rmtree(this['install']) shutil.rmtree(this['build']) utils.set_mtime_recursively(this['sandbox']) utils.make_deterministic_tar_archive(cachefile, this['sandbox']) shutil.move('%s.tar' % cachefile, cachefile) else: utils.set_mtime_recursively(this['install']) utils.make_deterministic_gztar_archive(cachefile, this['install']) shutil.move('%s.tar.gz' % cachefile, cachefile) app.config['counter'].increment() unpack(defs, this, cachefile) if app.config.get('kbas-password', 'insecure') != 'insecure' and \ app.config.get('kbas-url') is not None: if this.get('kind', 'chunk') in ['chunk', 'stratum']: with app.timer(this, 'upload'): upload(defs, this)
def cache(defs, this): if get_cache(defs, this): app.log(this, "Bah! I could have cached", cache_key(defs, this)) return tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(defs, this)) if this.get('kind') == "system": utils.hardlink_all_files(this['install'], this['sandbox']) shutil.rmtree(this['install']) shutil.rmtree(this['build']) utils.set_mtime_recursively(this['sandbox']) utils.make_deterministic_tar_archive(cachefile, this['sandbox']) os.rename('%s.tar' % cachefile, cachefile) else: utils.set_mtime_recursively(this['install']) utils.make_deterministic_gztar_archive(cachefile, this['install']) os.rename('%s.tar.gz' % cachefile, cachefile) unpack(defs, this, cachefile) if app.config.get('kbas-password', 'insecure') != 'insecure' and \ app.config.get('kbas-url', 'http://foo.bar/') != 'http://foo.bar/': if this.get('kind', 'chunk') == 'chunk': with app.timer(this, 'upload'): upload(defs, this)
def cache(defs, this, full_root=False): if get_cache(defs, this): app.log(this, "Bah! I could have cached", cache_key(defs, this)) return tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(defs, this)) if full_root: utils.hardlink_all_files(this['install'], this['sandbox']) shutil.rmtree(this['install']) shutil.rmtree(this['build']) utils.set_mtime_recursively(this['sandbox']) utils.make_deterministic_tar_archive(cachefile, this['sandbox']) os.rename('%s.tar' % cachefile, cachefile) else: utils.set_mtime_recursively(this['install']) utils.make_deterministic_gztar_archive(cachefile, this['install']) os.rename('%s.tar.gz' % cachefile, cachefile) unpackdir = cachefile + '.unpacked' os.makedirs(unpackdir) if call(['tar', 'xf', cachefile, '--directory', unpackdir]): app.exit(this, 'ERROR: Problem unpacking', cachefile) try: target = os.path.join(app.config['artifacts'], cache_key(defs, this)) os.rename(tmpdir, target) size = os.path.getsize(get_cache(defs, this)) app.log(this, 'Now cached %s bytes as' % size, cache_key(defs, this)) except: app.log(this, 'Bah! I raced and rebuilt', cache_key(defs, this))
def install_artifact(this, component, permit_bootstrap=True): component = Definitions().get(component) if component.get('build-mode') == 'bootstrap' and permit_bootstrap == False: return app.log(this, 'Installing %s' % component['cache']) unpackdir = cache.unpack(component) utils.hardlink_all_files(unpackdir, this['assembly'])
def install(defs, this, component): if os.path.exists(os.path.join(this['sandbox'], 'baserock', component['name'] + '.meta')): return if app.config.get('log-verbose'): app.log(this, 'Installing %s' % component['cache']) unpackdir = cache.get_cache(defs, component) + '.unpacked' if this.get('kind') is 'system': utils.copy_all_files(unpackdir, this['sandbox']) else: utils.hardlink_all_files(unpackdir, this['sandbox'])
def install(defs, this, component): # populate this['sandbox'] with the artifact files from component if os.path.exists(os.path.join(this["sandbox"], "baserock", component["name"] + ".meta")): return if app.config.get("log-verbose"): app.log(this, "Installing %s" % component["cache"]) if cache.get_cache(defs, component) is False: app.exit(this, "ERROR: unable to get cache for", component["name"]) unpackdir = cache.get_cache(defs, component) + ".unpacked" if this.get("kind") is "system": utils.copy_all_files(unpackdir, this["sandbox"]) else: utils.hardlink_all_files(unpackdir, this["sandbox"])
def install(dn, component): # populate dn['sandbox'] with the artifact files from component if os.path.exists(os.path.join(dn['sandbox'], 'baserock', component['name'] + '.meta')): return app.log(dn, 'Sandbox: installing %s' % component['cache'], verbose=True) if cache.get_cache(component) is False: app.log(dn, 'Unable to get cache for', component['name'], exit=True) unpackdir = cache.get_cache(component) + '.unpacked' if dn.get('kind') is 'system': utils.copy_all_files(unpackdir, dn['sandbox']) else: utils.hardlink_all_files(unpackdir, dn['sandbox'])
def install(defs, this, component): # populate this['sandbox'] with the artifact files from component if os.path.exists(os.path.join(this['sandbox'], 'baserock', component['name'] + '.meta')): return if app.config.get('log-verbose'): app.log(this, 'Sandbox: installing %s' % component['cache']) if cache.get_cache(defs, component) is False: app.exit(this, 'ERROR: unable to get cache for', component['name']) unpackdir = cache.get_cache(defs, component) + '.unpacked' if this.get('kind') is 'system': utils.copy_all_files(unpackdir, this['sandbox']) else: utils.hardlink_all_files(unpackdir, this['sandbox'])
def install(dn, component): # populate dn['sandbox'] with the artifact files from component if os.path.exists( os.path.join(dn['sandbox'], 'baserock', component['name'] + '.meta')): return app.log(dn, 'Sandbox: installing %s' % component['cache'], verbose=True) if cache.get_cache(component) is False: app.log(dn, 'Unable to get cache for', component['name'], exit=True) unpackdir = cache.get_cache(component) + '.unpacked' if dn.get('kind') is 'system': utils.copy_all_files(unpackdir, dn['sandbox']) else: utils.hardlink_all_files(unpackdir, dn['sandbox'])
def install(defs, this, component): # populate this['sandbox'] with the artifact files from component if os.path.exists( os.path.join(this['sandbox'], 'baserock', component['name'] + '.meta')): return if app.config.get('log-verbose'): app.log(this, 'Installing %s' % component['cache']) if cache.get_cache(defs, component) is False: app.exit(this, 'ERROR: unable to get cache for', component['name']) unpackdir = cache.get_cache(defs, component) + '.unpacked' if this.get('kind') is 'system': utils.copy_all_files(unpackdir, this['sandbox']) else: utils.hardlink_all_files(unpackdir, this['sandbox'])
def _install(defs, this, component): if os.path.exists(os.path.join(this['sandbox'], 'baserock', component['name'] + '.meta')): return for it in component.get('build-depends', []): dependency = defs.get(it) if (dependency.get('build-mode', 'staging') == component.get('build-mode', 'staging')): _install(defs, this, dependency) for it in component.get('contents', []): subcomponent = defs.get(it) if subcomponent.get('build-mode', 'staging') != 'bootstrap': _install(defs, this, subcomponent) unpackdir = cache.unpack(defs, component) if this.get('kind') is 'system': utils.copy_all_files(unpackdir, this['sandbox']) else: utils.hardlink_all_files(unpackdir, this['sandbox'])