def deploy(target):
'''Deploy systems and subsystems recursively'''
defs = Definitions()
deployment = target if type(target) is dict else defs.get(target)
with app.timer(deployment, 'Starting deployment'):
for system in deployment.get('systems', []):
deploy(system)
for subsystem in system.get('subsystems', []):
deploy(subsystem)
system = defs.get(deployment['path'])
if system.get('arch') and system['arch'] != app.settings['arch']:
app.log(target, 'Skipping deployment for', system['arch'])
return None
sandbox.setup(system)
for name, deployment in deployment.get('deploy', {}).iteritems():
method = os.path.basename(deployment['type'])
sandbox.run_extension(system, deployment, 'check', method)
app.log(system, "Extracting system artifact")
with open(cache.get_cache(system), "r") as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
sandbox.remove(system)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
# needed for artifact splitting
load_manifest(defs, target)
return cache.cache_key(defs, target)
random.seed(datetime.datetime.now())
component = defs.get(target)
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'assembly'):
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble_system_recursively(system)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
splits = None
if component.get('kind') == 'system':
splits = subcomponent.get('artifacts')
sandbox.install(defs, component, subcomponent, splits)
app.config['counter'] += 1
if 'systems' not in component:
with app.timer(component, 'build'):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(defs, component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if cache_key(defs, component) is False:
return False
if get_cache(defs, component):
return cache_key(defs, component)
if app.config.get('kbas-url'):
with claim(defs, component):
if get_remote(defs, component):
app.config['counter'].increment()
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component and not get_cache(defs, component):
if app.config.get('instances', 1) > 1:
with claim(defs, component):
# in here, exceptions get eaten
do_build(defs, component)
else:
# in here, exceptions do not get eaten
do_build(defs, component)
app.remove_dir(component['sandbox'])
return cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if cache_key(defs, component) is False:
return False
if get_cache(defs, component):
return cache_key(defs, component)
if app.config.get('kbas-url'):
with claim(defs, component):
if get_remote(defs, component):
app.config['counter'].increment()
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component and not get_cache(defs, component):
if app.config.get('instances', 1) > 1:
with claim(defs, component):
# in here, exceptions get eaten
do_build(defs, component)
else:
# in here, exceptions do not get eaten
do_build(defs, component)
app.remove_dir(component['sandbox'])
return cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if get_cache(defs, component) or get_remote(defs, component):
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component:
if is_building(defs, component):
import time
time.sleep(10)
raise Exception
app.config['counter'] += 1
if not get_cache(defs, component):
with app.timer(component, 'build of %s' % component['cache']):
with claim(defs, component):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
sandbox.remove(component)
return cache_key(defs, component)
def deploy_system(defs, system_spec, parent_location=''):
'''Deploy a system and subsystems recursively.
Takes a system spec (i.e. an entry in the "systems" list in a cluster
definition), and optionally a path to a parent system tree. If
`parent_location` is given then the `location` given in the cluster
definition for the subsystem is appended to `parent_location`, with
the result being used as the location for the deployment extensions.
'''
system = defs.get(system_spec['path'])
deploy_defaults = system_spec.get('deploy-defaults')
if system.get('arch') and system['arch'] != app.settings['arch']:
app.log(system, 'Skipping deployment for', system['arch'])
return None
sandbox.setup(system)
app.log(system, 'Extracting system artifact into', system['sandbox'])
with open(cache.get_cache(defs, system), 'r') as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for subsystem_spec in system_spec.get('subsystems', []):
if deploy_defaults:
subsystem_spec = dict(deploy_defaults.items()
+ subsystem_spec.items())
deploy_system(defs, subsystem_spec, parent_location=system['sandbox'])
for name, deployment in system_spec.get('deploy', {}).iteritems():
method = os.path.basename(deployment['type'])
if deploy_defaults:
deployment = dict(deploy_defaults.items()
+ deployment.items())
do_deployment_manifest(system, deployment)
if parent_location:
deployment['location'] = os.path.join(
parent_location, deployment['location'].lstrip('/'))
try:
sandbox.run_extension(system, deployment, 'check', method)
except KeyError:
app.log(system, "Couldn't find a check extension for",
method)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
sandbox.remove(system)
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return
defs = Definitions()
this = defs.get(target)
with app.timer(this, 'Starting assembly'):
with sandbox.setup(this):
for it in this.get('build-depends', []):
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
for it in this.get('contents', []):
component = defs.get(it)
if component.get('build-mode') == 'bootstrap':
continue
assemble(component)
sandbox.install(this, component)
if this.get('build-mode') != 'bootstrap':
sandbox.ldconfig(this)
else:
app.log(this, "No ldconfig because bootstrap mode is engaged")
build(this)
if this.get('devices'):
sandbox.create_devices(this)
cache.cache(this)
def compose(defs, target):
'''Work through defs tree, building and assembling until target exists'''
component = defs.get(target)
if app.config.get('log-verbose'):
app.log(target, "Composing", component['name'])
# if we can't calculate cache key, we can't create this component
if cache_key(defs, component) is False:
return False
# if this component is already cached, we're done
if get_cache(defs, component):
return cache_key(defs, component)
# if we have a kbas, look there to see if this component exists
if app.config.get('kbas-url'):
with claim(defs, component):
if get_remote(defs, component):
app.config['counter'].increment()
return cache_key(defs, component)
if component.get('arch') and component['arch'] != app.config['arch']:
return None
with sandbox.setup(component):
assemble(defs, component)
if 'systems' not in component and not get_cache(defs, component):
install_dependencies(defs, component)
build(defs, component)
return cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
return cache.cache_key(defs, target)
component = defs.get(target)
if component.get('arch') and component['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'Starting assembly'):
sandbox.setup(component)
for system_spec in component.get('systems', []):
assemble_system_recursively(system_spec)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
sandbox.install(defs, component, subcomponent)
app.settings['counter'] += 1
if 'systems' not in component:
build(defs, component)
do_manifest(component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def deploy_system(defs, system_spec, parent_location=''):
'''Deploy a system and subsystems recursively.
Takes a system spec (i.e. an entry in the "systems" list in a cluster
definition), and optionally a path to a parent system tree. If
`parent_location` is given then the `location` given in the cluster
definition for the subsystem is appended to `parent_location`, with
the result being used as the location for the deployment extensions.
'''
system = defs.get(system_spec['path'])
deploy_defaults = system_spec.get('deploy-defaults')
sandbox.setup(system)
app.log(system, 'Extracting system artifact into', system['sandbox'])
with open(cache.get_cache(defs, system), 'r') as artifact:
call(['tar', 'x', '--directory', system['sandbox']], stdin=artifact)
for subsystem in system_spec.get('subsystems', []):
if deploy_defaults:
subsystem = dict(deploy_defaults.items() + subsystem.items())
deploy_system(defs, subsystem, parent_location=system['sandbox'])
for name, deployment in system_spec.get('deploy', {}).iteritems():
method = deployment.get('type') or deployment.get('upgrade-type')
method = os.path.basename(method)
if deploy_defaults:
deployment = dict(deploy_defaults.items() + deployment.items())
do_deployment_manifest(system, deployment)
if parent_location:
location = deployment.get('location') or \
deployment.get('upgrade-location')
deployment['location'] = os.path.join(parent_location,
location.lstrip('/'))
try:
sandbox.run_extension(system, deployment, 'check', method)
except KeyError:
app.log(system, "Couldn't find a check extension for", method)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
app.remove_dir(system['sandbox'])
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return cache.cache_key(target)
defs = Definitions()
this = defs.get(target)
if this.get('arch') and this['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', this['arch'])
return None
with app.timer(this, 'Starting assembly'):
sandbox.setup(this)
for it in this.get('systems', []):
system = defs.get(it)
assemble(system)
for subsystem in this.get('subsystems', []):
assemble(subsystem)
dependencies = this.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
contents = this.get('contents', [])
random.shuffle(contents)
for it in contents:
component = defs.get(it)
if component.get('build-mode') != 'bootstrap':
assemble(component)
sandbox.install(this, component)
build(this)
do_manifest(this)
cache.cache(this, full_root=this.get('kind', None) == "system")
sandbox.remove(this)
return cache.cache_key(this)
def deploy_system(system_spec, parent_location=''):
'''Deploy a system and subsystems recursively.
Takes a system spec (i.e. an entry in the "systems" list in a cluster
definition), and optionally a path to a parent system tree. If
`parent_location` is given then the `location` given in the cluster
definition for the subsystem is appended to `parent_location`, with
the result being used as the location for the deployment extensions.
'''
system = app.defs.get(system_spec['path'])
if not cache.get_cache(system):
app.log('DEPLOY', 'System is not built, cannot deploy:\n', system,
exit=True)
deploy_defaults = system_spec.get('deploy-defaults')
with sandbox.setup(system):
app.log(system, 'Extracting system artifact into', system['sandbox'])
with open(cache.get_cache(system), 'r') as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for subsystem in system_spec.get('subsystems', []):
if deploy_defaults:
subsystem = dict(deploy_defaults.items() + subsystem.items())
deploy_system(subsystem, parent_location=system['sandbox'])
for name, deployment in system_spec.get('deploy', {}).iteritems():
method = deployment.get('type') or deployment.get('upgrade-type')
method = os.path.basename(method)
if deploy_defaults:
deployment = dict(deploy_defaults.items() + deployment.items())
do_deployment_manifest(system, deployment)
if parent_location:
for l in ['location', 'upgrade-location']:
if l in deployment:
dn = deployment[l].lstrip('/')
deployment[l] = os.path.join(parent_location, dn)
try:
sandbox.run_extension(system, deployment, 'check', method)
except KeyError:
app.log(system, "Couldn't find a check extension for", method)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
dn['tried'] = True
return False
# if dn is already cached, we're done
if get_cache(dn):
return cache_key(dn)
log(dn, "Composing", dn['name'], verbose=True)
# if we have a kbas, look there to see if this component exists
if config.get('kbas-url') and not config.get('reproduce'):
with claim(dn):
if get_remote(dn):
config['counter'].increment()
return cache_key(dn)
# we only work with user-specified arch
if 'arch' in dn and dn['arch'] != config['arch']:
return None
# Create composite components (strata, systems, clusters)
systems = dn.get('systems', [])
shuffle(systems)
for system in systems:
for s in system.get('subsystems', []):
subsystem = app.defs.get(s['path'])
compose(subsystem)
compose(system['path'])
with sandbox.setup(dn):
install_contents(dn)
build(dn) # bring in 'build-depends', and run make
return cache_key(dn)
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
dn['tried'] = True
return False
# if dn is already cached, we're done
if get_cache(dn):
return cache_key(dn)
log(dn, "Composing", dn['name'], verbose=True)
# if we have a kbas, look there to see if this component exists
if config.get('kbas-url') and not config.get('reproduce'):
with claim(dn):
if get_remote(dn):
config['counter'].increment()
return cache_key(dn)
# we only work with user-specified arch
if 'arch' in dn and dn['arch'] != config['arch']:
return None
# Create composite components (strata, systems, clusters)
systems = dn.get('systems', [])
shuffle(systems)
for system in systems:
for s in system.get('subsystems', []):
subsystem = app.defs.get(s['path'])
compose(subsystem)
compose(system['path'])
with sandbox.setup(dn):
install_contents(dn)
build(dn) # bring in 'build-depends', and run make
return cache_key(dn)
import sandbox
import backup_helper
from notify import send_mail
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Password Expiration Check')
parser.add_argument("--do",
default=True,
help='Send email',
action="store_false")
args = parser.parse_args()
dry_run = args.do
sandbox.setup()
syslog.openlog(ident='backup',
logoption=syslog.LOG_PID,
facility=syslog.LOG_LOCAL3)
content = []
if sys.stdout.isatty():
log_fp = sys.stdout
else:
log_fp = os.tmpfile()
try:
cfg_yaml = osp.join(osp.dirname(osp.realpath(__file__)),
"backup.yaml")