def do_build(defs, component):
app.config['counter'].increment()
with app.timer(component, 'build of %s' % component['cache']):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
def do_build(defs, component):
app.config['counter'].increment()
with app.timer(component, 'build of %s' % component['cache']):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
# needed for artifact splitting
load_manifest(defs, target)
return cache.cache_key(defs, target)
random.seed(datetime.datetime.now())
component = defs.get(target)
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'assembly'):
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble_system_recursively(system)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
splits = None
if component.get('kind') == 'system':
splits = subcomponent.get('artifacts')
sandbox.install(defs, component, subcomponent, splits)
app.config['counter'] += 1
if 'systems' not in component:
with app.timer(component, 'build'):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(defs, component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def build(defs, component):
'''Create an artifact for a single component and add it to the cache'''
with claim(defs, component):
app.config['counter'].increment()
with app.timer(component, 'build of %s' % component['cache']):
run_build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if get_cache(defs, component) or get_remote(defs, component):
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component:
if is_building(defs, component):
import time
time.sleep(10)
raise Exception
app.config['counter'] += 1
if not get_cache(defs, component):
with app.timer(component, 'build of %s' % component['cache']):
with claim(defs, component):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
sandbox.remove(component)
return cache_key(defs, component)
def build(defs, component):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(defs, component):
return
with claim(defs, component):
if component.get('kind', 'chunk') == 'chunk':
install_dependencies(defs, component)
with timer(component, 'build of %s' % component['cache']):
run_build(defs, component)
with timer(component, 'artifact creation'):
write_metadata(defs, component)
cache(defs, component)
def build(defs, component):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(defs, component):
return
with claim(defs, component):
if component.get('kind', 'chunk') == 'chunk':
install_dependencies(defs, component)
with timer(component, 'build of %s' % component['cache']):
run_build(defs, component)
with timer(component, 'artifact creation'):
write_metadata(defs, component)
cache(defs, component)
def deploy(target):
'''Deploy systems and subsystems recursively'''
defs = Definitions()
deployment = target if type(target) is dict else defs.get(target)
with app.timer(deployment, 'Starting deployment'):
for system in deployment.get('systems', []):
deploy(system)
for subsystem in system.get('subsystems', []):
deploy(subsystem)
system = defs.get(deployment['path'])
if system.get('arch') and system['arch'] != app.settings['arch']:
app.log(target, 'Skipping deployment for', system['arch'])
return None
sandbox.setup(system)
for name, deployment in deployment.get('deploy', {}).iteritems():
method = os.path.basename(deployment['type'])
sandbox.run_extension(system, deployment, 'check', method)
app.log(system, "Extracting system artifact")
with open(cache.get_cache(system), "r") as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
sandbox.remove(system)
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return
defs = Definitions()
this = defs.get(target)
with app.timer(this, 'Starting assembly'):
with sandbox.setup(this):
for it in this.get('build-depends', []):
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
for it in this.get('contents', []):
component = defs.get(it)
if component.get('build-mode') == 'bootstrap':
continue
assemble(component)
sandbox.install(this, component)
if this.get('build-mode') != 'bootstrap':
sandbox.ldconfig(this)
else:
app.log(this, "No ldconfig because bootstrap mode is engaged")
build(this)
if this.get('devices'):
sandbox.create_devices(this)
cache.cache(this)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get('kind') == "system":
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
shutil.move('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
shutil.move('%s.tar.gz' % cachefile, cachefile)
app.config['counter'].increment()
unpack(defs, this, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url') is not None:
if this.get('kind', 'chunk') in ['chunk', 'stratum']:
with app.timer(this, 'upload'):
upload(defs, this)
def deploy(defs, target):
'''Deploy a cluster definition.'''
arch = app.config['arch']
for system in target.get('systems', []):
if defs.get(system).get('arch', arch) == arch:
with app.timer(system, 'deployment'):
deploy_system(defs, system)
def cache(dn):
if get_cache(dn):
app.log(dn, "Bah! I could have cached", cache_key(dn))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(dn))
if dn.get('kind') == "system":
utils.hardlink_all_files(dn['install'], dn['sandbox'])
shutil.rmtree(dn['checkout'])
utils.set_mtime_recursively(dn['install'])
utils.make_deterministic_tar_archive(cachefile, dn['install'])
shutil.move('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(dn['install'])
utils.make_deterministic_gztar_archive(cachefile, dn['install'])
shutil.move('%s.tar.gz' % cachefile, cachefile)
app.config['counter'].increment()
unpack(dn, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url') is not None:
if dn.get('kind', 'chunk') in app.config.get('kbas-upload', 'chunk'):
with app.timer(dn, 'upload'):
upload(dn)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get('kind') == "system":
utils.hardlink_all_files(this['install'], this['sandbox'])
shutil.rmtree(this['install'])
shutil.rmtree(this['build'])
utils.set_mtime_recursively(this['sandbox'])
utils.make_deterministic_tar_archive(cachefile, this['sandbox'])
os.rename('%s.tar' % cachefile, cachefile)
else:
utils.set_mtime_recursively(this['install'])
utils.make_deterministic_gztar_archive(cachefile, this['install'])
os.rename('%s.tar.gz' % cachefile, cachefile)
unpack(defs, this, cachefile)
if app.config.get('kbas-password', 'insecure') != 'insecure' and \
app.config.get('kbas-url', 'http://foo.bar/') != 'http://foo.bar/':
if this.get('kind', 'chunk') == 'chunk':
with app.timer(this, 'upload'):
upload(defs, this)
def cache(defs, this):
if get_cache(defs, this):
app.log(this, "Bah! I could have cached", cache_key(defs, this))
return
tempfile.tempdir = app.config["tmp"]
tmpdir = tempfile.mkdtemp()
cachefile = os.path.join(tmpdir, cache_key(defs, this))
if this.get("kind") == "system":
utils.hardlink_all_files(this["install"], this["sandbox"])
shutil.rmtree(this["install"])
shutil.rmtree(this["build"])
utils.set_mtime_recursively(this["sandbox"])
utils.make_deterministic_tar_archive(cachefile, this["sandbox"])
os.rename("%s.tar" % cachefile, cachefile)
else:
utils.set_mtime_recursively(this["install"])
utils.make_deterministic_gztar_archive(cachefile, this["install"])
os.rename("%s.tar.gz" % cachefile, cachefile)
unpack(defs, this, cachefile)
if (
app.config.get("kbas-password", "insecure") != "insecure"
and app.config.get("kbas-url", "http://foo.bar/") != "http://foo.bar/"
):
if this.get("kind") is not "cluster":
with app.timer(this, "upload"):
upload(defs, this)
def deploy(defs, target):
'''Deploy a cluster definition.'''
deployment = target if type(target) is dict else defs.get(target)
with app.timer(deployment, 'Starting deployment'):
for system in deployment.get('systems', []):
deploy_system(defs, system)
def build(dn):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(dn):
return
with claim(dn):
if dn.get('kind', 'chunk') == 'chunk':
install_dependencies(dn)
with timer(dn, 'build of %s' % dn['cache']):
run_build(dn)
with timer(dn, 'artifact creation'):
if dn.get('kind', 'chunk') == 'system':
install_split_artifacts(dn)
write_metadata(dn)
cache(dn)
def build(dn):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(dn):
return
with claim(dn):
if dn.get('kind', 'chunk') == 'chunk':
install_dependencies(dn)
with timer(dn, 'build of %s' % dn['cache']):
run_build(dn)
with timer(dn, 'artifact creation'):
if dn.get('kind', 'chunk') == 'system':
install_split_artifacts(dn)
write_metadata(dn)
cache(dn)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
return cache.cache_key(defs, target)
component = defs.get(target)
if component.get('arch') and component['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'Starting assembly'):
sandbox.setup(component)
for system_spec in component.get('systems', []):
assemble_system_recursively(system_spec)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
sandbox.install(defs, component, subcomponent)
app.settings['counter'] += 1
if 'systems' not in component:
build(defs, component)
do_manifest(component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return cache.cache_key(target)
defs = Definitions()
this = defs.get(target)
if this.get('arch') and this['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', this['arch'])
return None
with app.timer(this, 'Starting assembly'):
sandbox.setup(this)
for it in this.get('systems', []):
system = defs.get(it)
assemble(system)
for subsystem in this.get('subsystems', []):
assemble(subsystem)
dependencies = this.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
contents = this.get('contents', [])
random.shuffle(contents)
for it in contents:
component = defs.get(it)
if component.get('build-mode') != 'bootstrap':
assemble(component)
sandbox.install(this, component)
build(this)
do_manifest(this)
cache.cache(this, full_root=this.get('kind', None) == "system")
sandbox.remove(this)
return cache.cache_key(this)
import sandbox
import sandboxlib
print('')
if not os.path.exists('./VERSION'):
if os.path.basename(os.getcwd()) != 'definitions':
if os.path.isdir(os.path.join(os.getcwd(), 'definitions')):
os.chdir(os.path.join(os.getcwd(), 'definitions'))
else:
if os.path.isdir(os.path.join(os.getcwd(), '..', 'definitions')):
os.chdir(os.path.join(os.getcwd(), '..', 'definitions'))
app.setup(sys.argv)
app.cleanup(app.config['tmp'])
with app.timer('TOTAL'):
tmp_lock = open(os.path.join(app.config['tmp'], 'lock'), 'r')
fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
target = os.path.join(app.config['defdir'], app.config['target'])
app.log('TARGET', 'Target is %s' % target, app.config['arch'])
with app.timer('DEFINITIONS', 'parsing %s' % app.config['def-version']):
defs = Definitions()
with app.timer('CACHE-KEYS', 'cache-key calculations'):
cache.cache_key(defs, app.config['target'])
cache.cull(app.config['artifacts'])
target = defs.get(app.config['target'])
if app.config['total'] == 0 or (app.config['total'] == 1
and target.get('kind') == 'cluster'):
app.exit('ARCH', 'ERROR: no definitions found for', app.config['arch'])
import sandboxlib
print('')
if not os.path.exists('./VERSION'):
if os.path.basename(os.getcwd()) != 'definitions':
if os.path.isdir(os.path.join(os.getcwd(), 'definitions')):
os.chdir(os.path.join(os.getcwd(), 'definitions'))
else:
if os.path.isdir(os.path.join(os.getcwd(), '..', 'definitions')):
os.chdir(os.path.join(os.getcwd(), '..', 'definitions'))
app.setup(sys.argv)
app.cleanup(app.config['tmp'])
with app.timer('TOTAL'):
tmp_lock = open(os.path.join(app.config['tmp'], 'lock'), 'r')
fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
target = os.path.join(app.config['defdir'], app.config['target'])
app.log('TARGET', 'Target is %s' % target, app.config['arch'])
with app.timer('DEFINITIONS', 'parsing %s' % app.config['def-version']):
defs = Definitions()
with app.timer('CACHE-KEYS', 'cache-key calculations'):
cache.cache_key(defs, app.config['target'])
cache.cull(app.config['artifacts'])
target = defs.get(app.config['target'])
if app.config['total'] == 0 or (app.config['total'] == 1 and
target.get('kind') == 'cluster'):
app.exit('ARCH', 'ERROR: no definitions found for', app.config['arch'])
component = defs.get(it)
if component.get('repo'):
log('AGGREGATE', 'Adding aggregate for', component['name'])
aggregate += [{'get': component['name']}]
else:
log('PASSED', 'Adding passed for', component['name'])
aggregate += [{'get': component['name']}]
passed += [component['name']]
plan = [{'task': 'Build', 'config': config}, {'aggregate': aggregate}]
job = {'name': target['name'], 'plan': plan, 'passed': passed}
pipeline = {'resources': inputs(defs, target), 'jobs': [job]}
output = './pipeline.yml'
with open(output, 'w') as f:
f.write(yaml.dump(pipeline, default_flow_style=False))
exit('CONCOURSE', 'pipeline is at', output)
setup(sys.argv)
with timer('TOTAL'):
target = os.path.join(config['defdir'], config['target'])
log('TARGET', 'Target is %s' % target, config['arch'])
with timer('DEFINITIONS', 'parsing %s' % config['def-version']):
defs = Definitions()
with timer('CACHE-KEYS', 'cache-key calculations'):
cache.cache_key(defs, config['target'])
write_pipeline(defs, config['target'])
import sandboxlib
import os
import sys
import app
from assembly import assemble, deploy
from definitions import Definitions
import cache
import sandbox
print('')
app.setup(sys.argv)
with app.timer('TOTAL', '%s starts, version %s' % (app.settings['program'],
app.settings['program-version'])):
target = os.path.join(app.settings['defdir'], app.settings['target'])
app.log('TARGET', 'Target is %s' % target, app.settings['arch'])
with app.timer('DEFINITIONS', 'Parsing %s' % app.settings['def-version']):
defs = Definitions()
with app.timer('CACHE-KEYS', 'Calculating'):
cache.get_cache(defs, app.settings['target'])
defs.save_trees()
sandbox.executor = sandboxlib.executor_for_platform()
app.log(app.settings['target'], 'Sandbox using %s' % sandbox.executor)
assemble(defs, app.settings['target'])
deploy(defs, app.settings['target'])
component = defs.get(it)
if component.get('repo'):
log('AGGREGATE', 'Adding aggregate for', component['name'])
aggregate += [{'get': component['name']}]
else:
log('PASSED', 'Adding passed for', component['name'])
aggregate += [{'get': component['name']}]
passed += [component['name']]
plan = [{'task': 'Build', 'config': config}, {'aggregate': aggregate}]
job = {'name': target['name'], 'plan': plan, 'passed': passed}
pipeline = {'resources': inputs(defs, target), 'jobs': [job]}
output = './pipeline.yml'
with open(output, 'w') as f:
f.write(yaml.dump(pipeline, default_flow_style=False))
exit('CONCOURSE', 'pipeline is at', output)
setup(sys.argv)
with timer('TOTAL'):
target = os.path.join(config['defdir'], config['target'])
log('TARGET', 'Target is %s' % target, config['arch'])
with timer('DEFINITIONS', 'parsing %s' % config['def-version']):
defs = Definitions()
with timer('CACHE-KEYS', 'cache-key calculations'):
cache.cache_key(defs, config['target'])
write_pipeline(defs, config['target'])
target = sys.argv[1]
if len(sys.argv) == 3:
arch = sys.argv[2]
else:
arch = platform.machine()
if arch in ('mips', 'mips64'):
if arch == 'mips':
arch = 'mips32'
if sys.byteorder == 'big':
arch = arch + 'b'
else:
arch = arch + 'l'
with app.setup(target, arch):
with app.timer('TOTAL', 'ybd starts, version %s' %
app.settings['ybd-version']):
app.log('TARGET', 'Target is %s' % os.path.join(app.settings['defdir'],
target), arch)
with app.timer('DEFINITIONS', 'Parsing %s' % app.settings['def-ver']):
defs = Definitions()
with app.timer('CACHE-KEYS', 'Calculating'):
cache.get_cache(defs, app.settings['target'])
defs.save_trees()
sandbox.executor = sandboxlib.executor_for_platform()
app.log(target, 'Using %s for sandboxing' % sandbox.executor)
assemble(defs, app.settings['target'])
deploy(defs, app.settings['target'])
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# =*= License: GPL-2 =*=
'''A module to build a definition.'''
import os
import sys
from definitions import Definitions
import cache
import app
from assembly import assemble
import sandbox
target = os.path.splitext(os.path.basename(sys.argv[1]))[0]
arch = sys.argv[2]
with app.setup(target, arch):
with app.timer('TOTAL', 'YBD starts'):
defs = Definitions()
definition = defs.get(target)
with app.timer('CACHE-KEYS', 'Calculating'):
cache.get_cache(target)
defs.save_trees()
assemble(definition)
