def deploy(target):
'''Deploy systems and subsystems recursively'''
defs = Definitions()
deployment = target if type(target) is dict else defs.get(target)
with app.timer(deployment, 'Starting deployment'):
for system in deployment.get('systems', []):
deploy(system)
for subsystem in system.get('subsystems', []):
deploy(subsystem)
system = defs.get(deployment['path'])
if system.get('arch') and system['arch'] != app.settings['arch']:
app.log(target, 'Skipping deployment for', system['arch'])
return None
sandbox.setup(system)
for name, deployment in deployment.get('deploy', {}).iteritems():
method = os.path.basename(deployment['type'])
sandbox.run_extension(system, deployment, 'check', method)
app.log(system, "Extracting system artifact")
with open(cache.get_cache(system), "r") as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
sandbox.remove(system)
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return
defs = Definitions()
this = defs.get(target)
with app.timer(this, 'Starting assembly'):
with sandbox.setup(this):
for it in this.get('build-depends', []):
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
for it in this.get('contents', []):
component = defs.get(it)
if component.get('build-mode') == 'bootstrap':
continue
assemble(component)
sandbox.install(this, component)
if this.get('build-mode') != 'bootstrap':
sandbox.ldconfig(this)
else:
app.log(this, "No ldconfig because bootstrap mode is engaged")
build(this)
if this.get('devices'):
sandbox.create_devices(this)
cache.cache(this)
def install(this, component, permit_bootstrap=True):
component = Definitions().get(component)
for subcomponent in component.get('contents', []):
if component.get('build-mode') == 'bootstrap':
continue
install(this, subcomponent, False)
for dependency in component.get('build-depends', []):
install(this, dependency, False)
install_artifact(this, component, permit_bootstrap)
def install_artifact(this, component, permit_bootstrap=True):
component = Definitions().get(component)
if component.get('build-mode') == 'bootstrap' and permit_bootstrap == False:
return
app.log(this, 'Installing %s' % component['cache'])
unpackdir = cache.unpack(component)
utils.hardlink_all_files(unpackdir, this['assembly'])
def clean_env(this):
env = {}
extra_path = []
defs = Definitions()
if app.settings['no-ccache']:
ccache_path = []
else:
ccache_path = ['/usr/lib/ccache']
env['CCACHE_DIR'] = '/tmp/ccache'
env['CCACHE_EXTRAFILES'] = ':'.join(
f for f in ('/baserock/binutils.meta',
'/baserock/eglibc.meta',
'/baserock/gcc.meta') if os.path.exists(f))
if not app.settings.get('no-distcc'):
env['CCACHE_PREFIX'] = 'distcc'
prefixes = []
for name in this.get('build-depends', []):
dependency = defs.get(name)
prefixes.append(dependency.get('prefix'))
prefixes = set(prefixes)
for prefix in prefixes:
if prefix:
bin_path = os.path.join(prefix, 'bin')
extra_path += [bin_path]
if this.get('build-mode') == 'bootstrap':
rel_path = extra_path + ccache_path
full_path = [os.path.normpath(this['assembly'] + p)
for p in rel_path]
path = full_path + app.settings['base-path']
env['DESTDIR'] = this.get('install')
else:
path = extra_path + ccache_path + app.settings['base-path']
env['DESTDIR'] = os.path.join('/',
os.path.basename(this.get('install')))
env['PATH'] = ':'.join(path)
env['PREFIX'] = this.get('prefix') or '/usr'
env['MAKEFLAGS'] = '-j%s' % (this.get('max-jobs') or
app.settings['max_jobs'])
env['TERM'] = 'dumb'
env['SHELL'] = '/bin/sh'
env['USER'] = env['USERNAME'] = env['LOGNAME'] = 'tomjon'
env['LC_ALL'] = 'C'
env['HOME'] = '/tmp'
arch = app.settings['arch']
cpu = 'i686' if arch == 'x86_32' else arch
abi = 'eabi' if arch.startswith('arm') else ''
env['TARGET'] = cpu + '-baserock-linux-gnu' + abi
env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi
env['MORPH_ARCH'] = arch
return env
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return cache.cache_key(target)
defs = Definitions()
this = defs.get(target)
if this.get('arch') and this['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', this['arch'])
return None
with app.timer(this, 'Starting assembly'):
sandbox.setup(this)
for it in this.get('systems', []):
system = defs.get(it)
assemble(system)
for subsystem in this.get('subsystems', []):
assemble(subsystem)
dependencies = this.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
contents = this.get('contents', [])
random.shuffle(contents)
for it in contents:
component = defs.get(it)
if component.get('build-mode') != 'bootstrap':
assemble(component)
sandbox.install(this, component)
build(this)
do_manifest(this)
cache.cache(this, full_root=this.get('kind', None) == "system")
sandbox.remove(this)
return cache.cache_key(this)
def _install(this, component):
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
for it in component.get('build-depends', []):
dependency = Definitions().get(it)
if (dependency.get('build-mode', 'staging') ==
component.get('build-mode', 'staging')):
_install(this, dependency)
for it in component.get('contents', []):
subcomponent = Definitions().get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
_install(this, subcomponent)
unpackdir = cache.unpack(component)
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
app.setup(sys.argv)
app.cleanup(app.config['tmp'])
with app.timer('TOTAL'):
tmp_lock = open(os.path.join(app.config['tmp'], 'lock'), 'r')
fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
target = os.path.join(app.config['defdir'], app.config['target'])
app.log('TARGET', 'Target is %s' % target, app.config['arch'])
with app.timer('DEFINITIONS', 'parsing %s' % app.config['def-version']):
defs = Definitions()
with app.timer('CACHE-KEYS', 'cache-key calculations'):
cache.cache_key(defs, app.config['target'])
cache.cull(app.config['artifacts'])
target = defs.get(app.config['target'])
if app.config['total'] == 0 or (app.config['total'] == 1 and
target.get('kind') == 'cluster'):
app.exit('ARCH', 'ERROR: no definitions found for', app.config['arch'])
defs.save_trees()
if app.config.get('mode', 'normal') == 'keys-only':
with open('./ybd.result', 'w') as f:
f.write(target['cache'] + '\n')
os._exit(0)
sandbox.executor = sandboxlib.executor_for_platform()
app.log(app.config['target'], 'Sandbox using %s' % sandbox.executor)
if sandboxlib.chroot == sandbox.executor:
app.log(app.config['target'], 'WARNING: using chroot is less safe ' +
'than using linux-user-chroot')
def env_vars_for_build(this):
env = {}
extra_path = []
defs = Definitions()
arch_dict = {
'i686': "x86_32",
'armv8l64': "aarch64",
'armv8b64': "aarch64_be"
}
if app.settings['no-ccache']:
ccache_path = []
else:
ccache_path = ['/usr/lib/ccache']
env['CCACHE_DIR'] = '/tmp/ccache'
env['CCACHE_EXTRAFILES'] = ':'.join(
f for f in ('/baserock/binutils.meta',
'/baserock/eglibc.meta',
'/baserock/gcc.meta') if os.path.exists(f))
if not app.settings.get('no-distcc'):
env['CCACHE_PREFIX'] = 'distcc'
prefixes = []
for name in this.get('build-depends', []):
dependency = defs.get(name)
prefixes.append(dependency.get('prefix', '/usr'))
prefixes = set(prefixes)
for prefix in prefixes:
if prefix:
bin_path = os.path.join(prefix, 'bin')
extra_path += [bin_path]
if this.get('build-mode') == 'bootstrap':
rel_path = extra_path + ccache_path
full_path = [os.path.normpath(this['sandbox'] + p)
for p in rel_path]
path = full_path + app.settings['base-path']
env['DESTDIR'] = this.get('install')
else:
path = extra_path + ccache_path + app.settings['base-path']
env['DESTDIR'] = os.path.join('/',
os.path.basename(this.get('install')))
env['PATH'] = ':'.join(path)
env['PREFIX'] = this.get('prefix') or '/usr'
env['MAKEFLAGS'] = '-j%s' % (this.get('max-jobs') or
app.settings['max-jobs'])
env['TERM'] = 'dumb'
env['SHELL'] = '/bin/sh'
env['USER'] = env['USERNAME'] = env['LOGNAME'] = 'tomjon'
env['LC_ALL'] = 'C'
env['HOME'] = '/tmp'
env['TZ'] = 'UTC'
arch = app.settings['arch']
cpu = arch_dict.get(arch, arch)
abi = 'eabi' if arch.startswith(('armv7', 'armv5')) else ''
env['TARGET'] = cpu + '-baserock-linux-gnu' + abi
env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi
env['MORPH_ARCH'] = arch
env['DEFINITIONS_REF'] = app.settings['def-ver']
env['YBD_REF'] = app.settings['ybd-version']
return env
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# =*= License: GPL-2 =*=
'''A module to build a definition.'''
import os
import sys
from definitions import Definitions
import cache
import app
from assembly import assemble
import sandbox
target = os.path.splitext(os.path.basename(sys.argv[1]))[0]
arch = sys.argv[2]
with app.setup(target, arch):
with app.timer('TOTAL', 'YBD starts'):
defs = Definitions()
definition = defs.get(target)
with app.timer('CACHE-KEYS', 'Calculating'):
cache.get_cache(target)
defs.save_trees()
assemble(definition)