def deploy(target): '''Deploy systems and subsystems recursively''' defs = Definitions() deployment = target if type(target) is dict else defs.get(target) with app.timer(deployment, 'Starting deployment'): for system in deployment.get('systems', []): deploy(system) for subsystem in system.get('subsystems', []): deploy(subsystem) system = defs.get(deployment['path']) if system.get('arch') and system['arch'] != app.settings['arch']: app.log(target, 'Skipping deployment for', system['arch']) return None sandbox.setup(system) for name, deployment in deployment.get('deploy', {}).iteritems(): method = os.path.basename(deployment['type']) sandbox.run_extension(system, deployment, 'check', method) app.log(system, "Extracting system artifact") with open(cache.get_cache(system), "r") as artifact: call(['tar', 'x', '--directory', system['sandbox']], stdin=artifact) for ext in system.get('configuration-extensions', []): sandbox.run_extension(system, deployment, 'configure', os.path.basename(ext)) os.chmod(system['sandbox'], 0o755) sandbox.run_extension(system, deployment, 'write', method) sandbox.remove(system)
def assemble(target): '''Assemble dependencies and contents recursively until target exists.''' if cache.get_cache(target): return defs = Definitions() this = defs.get(target) with app.timer(this, 'Starting assembly'): with sandbox.setup(this): for it in this.get('build-depends', []): dependency = defs.get(it) assemble(dependency) sandbox.install(this, dependency) for it in this.get('contents', []): component = defs.get(it) if component.get('build-mode') == 'bootstrap': continue assemble(component) sandbox.install(this, component) if this.get('build-mode') != 'bootstrap': sandbox.ldconfig(this) else: app.log(this, "No ldconfig because bootstrap mode is engaged") build(this) if this.get('devices'): sandbox.create_devices(this) cache.cache(this)
def install(this, component, permit_bootstrap=True): component = Definitions().get(component) for subcomponent in component.get('contents', []): if component.get('build-mode') == 'bootstrap': continue install(this, subcomponent, False) for dependency in component.get('build-depends', []): install(this, dependency, False) install_artifact(this, component, permit_bootstrap)
def install_artifact(this, component, permit_bootstrap=True): component = Definitions().get(component) if component.get('build-mode') == 'bootstrap' and permit_bootstrap == False: return app.log(this, 'Installing %s' % component['cache']) unpackdir = cache.unpack(component) utils.hardlink_all_files(unpackdir, this['assembly'])
def clean_env(this): env = {} extra_path = [] defs = Definitions() if app.settings['no-ccache']: ccache_path = [] else: ccache_path = ['/usr/lib/ccache'] env['CCACHE_DIR'] = '/tmp/ccache' env['CCACHE_EXTRAFILES'] = ':'.join( f for f in ('/baserock/binutils.meta', '/baserock/eglibc.meta', '/baserock/gcc.meta') if os.path.exists(f)) if not app.settings.get('no-distcc'): env['CCACHE_PREFIX'] = 'distcc' prefixes = [] for name in this.get('build-depends', []): dependency = defs.get(name) prefixes.append(dependency.get('prefix')) prefixes = set(prefixes) for prefix in prefixes: if prefix: bin_path = os.path.join(prefix, 'bin') extra_path += [bin_path] if this.get('build-mode') == 'bootstrap': rel_path = extra_path + ccache_path full_path = [os.path.normpath(this['assembly'] + p) for p in rel_path] path = full_path + app.settings['base-path'] env['DESTDIR'] = this.get('install') else: path = extra_path + ccache_path + app.settings['base-path'] env['DESTDIR'] = os.path.join('/', os.path.basename(this.get('install'))) env['PATH'] = ':'.join(path) env['PREFIX'] = this.get('prefix') or '/usr' env['MAKEFLAGS'] = '-j%s' % (this.get('max-jobs') or app.settings['max_jobs']) env['TERM'] = 'dumb' env['SHELL'] = '/bin/sh' env['USER'] = env['USERNAME'] = env['LOGNAME'] = 'tomjon' env['LC_ALL'] = 'C' env['HOME'] = '/tmp' arch = app.settings['arch'] cpu = 'i686' if arch == 'x86_32' else arch abi = 'eabi' if arch.startswith('arm') else '' env['TARGET'] = cpu + '-baserock-linux-gnu' + abi env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi env['MORPH_ARCH'] = arch return env
def assemble(target): '''Assemble dependencies and contents recursively until target exists.''' if cache.get_cache(target): return cache.cache_key(target) defs = Definitions() this = defs.get(target) if this.get('arch') and this['arch'] != app.settings['arch']: app.log(target, 'Skipping assembly for', this['arch']) return None with app.timer(this, 'Starting assembly'): sandbox.setup(this) for it in this.get('systems', []): system = defs.get(it) assemble(system) for subsystem in this.get('subsystems', []): assemble(subsystem) dependencies = this.get('build-depends', []) random.shuffle(dependencies) for it in dependencies: dependency = defs.get(it) assemble(dependency) sandbox.install(this, dependency) contents = this.get('contents', []) random.shuffle(contents) for it in contents: component = defs.get(it) if component.get('build-mode') != 'bootstrap': assemble(component) sandbox.install(this, component) build(this) do_manifest(this) cache.cache(this, full_root=this.get('kind', None) == "system") sandbox.remove(this) return cache.cache_key(this)
def _install(this, component): if os.path.exists(os.path.join(this['sandbox'], 'baserock', component['name'] + '.meta')): return for it in component.get('build-depends', []): dependency = Definitions().get(it) if (dependency.get('build-mode', 'staging') == component.get('build-mode', 'staging')): _install(this, dependency) for it in component.get('contents', []): subcomponent = Definitions().get(it) if subcomponent.get('build-mode', 'staging') != 'bootstrap': _install(this, subcomponent) unpackdir = cache.unpack(component) if this.get('kind') is 'system': utils.copy_all_files(unpackdir, this['sandbox']) else: utils.hardlink_all_files(unpackdir, this['sandbox'])
app.setup(sys.argv) app.cleanup(app.config['tmp']) with app.timer('TOTAL'): tmp_lock = open(os.path.join(app.config['tmp'], 'lock'), 'r') fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB) target = os.path.join(app.config['defdir'], app.config['target']) app.log('TARGET', 'Target is %s' % target, app.config['arch']) with app.timer('DEFINITIONS', 'parsing %s' % app.config['def-version']): defs = Definitions() with app.timer('CACHE-KEYS', 'cache-key calculations'): cache.cache_key(defs, app.config['target']) cache.cull(app.config['artifacts']) target = defs.get(app.config['target']) if app.config['total'] == 0 or (app.config['total'] == 1 and target.get('kind') == 'cluster'): app.exit('ARCH', 'ERROR: no definitions found for', app.config['arch']) defs.save_trees() if app.config.get('mode', 'normal') == 'keys-only': with open('./ybd.result', 'w') as f: f.write(target['cache'] + '\n') os._exit(0) sandbox.executor = sandboxlib.executor_for_platform() app.log(app.config['target'], 'Sandbox using %s' % sandbox.executor) if sandboxlib.chroot == sandbox.executor: app.log(app.config['target'], 'WARNING: using chroot is less safe ' + 'than using linux-user-chroot')
def env_vars_for_build(this): env = {} extra_path = [] defs = Definitions() arch_dict = { 'i686': "x86_32", 'armv8l64': "aarch64", 'armv8b64': "aarch64_be" } if app.settings['no-ccache']: ccache_path = [] else: ccache_path = ['/usr/lib/ccache'] env['CCACHE_DIR'] = '/tmp/ccache' env['CCACHE_EXTRAFILES'] = ':'.join( f for f in ('/baserock/binutils.meta', '/baserock/eglibc.meta', '/baserock/gcc.meta') if os.path.exists(f)) if not app.settings.get('no-distcc'): env['CCACHE_PREFIX'] = 'distcc' prefixes = [] for name in this.get('build-depends', []): dependency = defs.get(name) prefixes.append(dependency.get('prefix', '/usr')) prefixes = set(prefixes) for prefix in prefixes: if prefix: bin_path = os.path.join(prefix, 'bin') extra_path += [bin_path] if this.get('build-mode') == 'bootstrap': rel_path = extra_path + ccache_path full_path = [os.path.normpath(this['sandbox'] + p) for p in rel_path] path = full_path + app.settings['base-path'] env['DESTDIR'] = this.get('install') else: path = extra_path + ccache_path + app.settings['base-path'] env['DESTDIR'] = os.path.join('/', os.path.basename(this.get('install'))) env['PATH'] = ':'.join(path) env['PREFIX'] = this.get('prefix') or '/usr' env['MAKEFLAGS'] = '-j%s' % (this.get('max-jobs') or app.settings['max-jobs']) env['TERM'] = 'dumb' env['SHELL'] = '/bin/sh' env['USER'] = env['USERNAME'] = env['LOGNAME'] = 'tomjon' env['LC_ALL'] = 'C' env['HOME'] = '/tmp' env['TZ'] = 'UTC' arch = app.settings['arch'] cpu = arch_dict.get(arch, arch) abi = 'eabi' if arch.startswith(('armv7', 'armv5')) else '' env['TARGET'] = cpu + '-baserock-linux-gnu' + abi env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi env['MORPH_ARCH'] = arch env['DEFINITIONS_REF'] = app.settings['def-ver'] env['YBD_REF'] = app.settings['ybd-version'] return env
# This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # =*= License: GPL-2 =*= '''A module to build a definition.''' import os import sys from definitions import Definitions import cache import app from assembly import assemble import sandbox target = os.path.splitext(os.path.basename(sys.argv[1]))[0] arch = sys.argv[2] with app.setup(target, arch): with app.timer('TOTAL', 'YBD starts'): defs = Definitions() definition = defs.get(target) with app.timer('CACHE-KEYS', 'Calculating'): cache.get_cache(target) defs.save_trees() assemble(definition)