def compose(defs, target):
'''Work through defs tree, building and assembling until target exists'''
component = defs.get(target)
if app.config.get('log-verbose'):
app.log(target, "Composing", component['name'])
# if we can't calculate cache key, we can't create this component
if cache_key(defs, component) is False:
return False
# if this component is already cached, we're done
if get_cache(defs, component):
return cache_key(defs, component)
# if we have a kbas, look there to see if this component exists
if app.config.get('kbas-url'):
with claim(defs, component):
if get_remote(defs, component):
app.config['counter'].increment()
return cache_key(defs, component)
if component.get('arch') and component['arch'] != app.config['arch']:
return None
with sandbox.setup(component):
assemble(defs, component)
if 'systems' not in component and not get_cache(defs, component):
install_dependencies(defs, component)
build(defs, component)
return cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if cache_key(defs, component) is False:
return False
if get_cache(defs, component):
return cache_key(defs, component)
if app.config.get('kbas-url'):
with claim(defs, component):
if get_remote(defs, component):
app.config['counter'].increment()
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component and not get_cache(defs, component):
if app.config.get('instances', 1) > 1:
with claim(defs, component):
# in here, exceptions get eaten
do_build(defs, component)
else:
# in here, exceptions do not get eaten
do_build(defs, component)
app.remove_dir(component['sandbox'])
return cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if cache_key(defs, component) is False:
return False
if get_cache(defs, component):
return cache_key(defs, component)
if app.config.get('kbas-url'):
with claim(defs, component):
if get_remote(defs, component):
app.config['counter'].increment()
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component and not get_cache(defs, component):
if app.config.get('instances', 1) > 1:
with claim(defs, component):
# in here, exceptions get eaten
do_build(defs, component)
else:
# in here, exceptions do not get eaten
do_build(defs, component)
app.remove_dir(component['sandbox'])
return cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if get_cache(defs, component) or get_remote(defs, component):
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component:
if is_building(defs, component):
import time
time.sleep(10)
raise Exception
app.config['counter'] += 1
if not get_cache(defs, component):
with app.timer(component, 'build of %s' % component['cache']):
with claim(defs, component):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
sandbox.remove(component)
return cache_key(defs, component)
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
dn['tried'] = True
return False
# if dn is already cached, we're done
if get_cache(dn):
return cache_key(dn)
log(dn, "Composing", dn['name'], verbose=True)
# if we have a kbas, look there to see if this component exists
if config.get('kbas-url') and not config.get('reproduce'):
with claim(dn):
if get_remote(dn):
config['counter'].increment()
return cache_key(dn)
# we only work with user-specified arch
if 'arch' in dn and dn['arch'] != config['arch']:
return None
# Create composite components (strata, systems, clusters)
systems = dn.get('systems', [])
shuffle(systems)
for system in systems:
for s in system.get('subsystems', []):
subsystem = app.defs.get(s['path'])
compose(subsystem)
compose(system['path'])
with sandbox.setup(dn):
install_contents(dn)
build(dn) # bring in 'build-depends', and run make
return cache_key(dn)
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
dn['tried'] = True
return False
# if dn is already cached, we're done
if get_cache(dn):
return cache_key(dn)
log(dn, "Composing", dn['name'], verbose=True)
# if we have a kbas, look there to see if this component exists
if config.get('kbas-url') and not config.get('reproduce'):
with claim(dn):
if get_remote(dn):
config['counter'].increment()
return cache_key(dn)
# we only work with user-specified arch
if 'arch' in dn and dn['arch'] != config['arch']:
return None
# Create composite components (strata, systems, clusters)
systems = dn.get('systems', [])
shuffle(systems)
for system in systems:
for s in system.get('subsystems', []):
subsystem = app.defs.get(s['path'])
compose(subsystem)
compose(system['path'])
with sandbox.setup(dn):
install_contents(dn)
build(dn) # bring in 'build-depends', and run make
return cache_key(dn)