def deploy(target):
'''Deploy systems and subsystems recursively'''
defs = Definitions()
deployment = target if type(target) is dict else defs.get(target)
with app.timer(deployment, 'Starting deployment'):
for system in deployment.get('systems', []):
deploy(system)
for subsystem in system.get('subsystems', []):
deploy(subsystem)
system = defs.get(deployment['path'])
if system.get('arch') and system['arch'] != app.settings['arch']:
app.log(target, 'Skipping deployment for', system['arch'])
return None
sandbox.setup(system)
for name, deployment in deployment.get('deploy', {}).iteritems():
method = os.path.basename(deployment['type'])
sandbox.run_extension(system, deployment, 'check', method)
app.log(system, "Extracting system artifact")
with open(cache.get_cache(system), "r") as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
sandbox.remove(system)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
# needed for artifact splitting
load_manifest(defs, target)
return cache.cache_key(defs, target)
random.seed(datetime.datetime.now())
component = defs.get(target)
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'assembly'):
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble_system_recursively(system)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
splits = None
if component.get('kind') == 'system':
splits = subcomponent.get('artifacts')
sandbox.install(defs, component, subcomponent, splits)
app.config['counter'] += 1
if 'systems' not in component:
with app.timer(component, 'build'):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(defs, component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if get_cache(defs, component) or get_remote(defs, component):
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component:
if is_building(defs, component):
import time
time.sleep(10)
raise Exception
app.config['counter'] += 1
if not get_cache(defs, component):
with app.timer(component, 'build of %s' % component['cache']):
with claim(defs, component):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
sandbox.remove(component)
return cache_key(defs, component)
def deploy_system(defs, system_spec, parent_location=''):
'''Deploy a system and subsystems recursively.
Takes a system spec (i.e. an entry in the "systems" list in a cluster
definition), and optionally a path to a parent system tree. If
`parent_location` is given then the `location` given in the cluster
definition for the subsystem is appended to `parent_location`, with
the result being used as the location for the deployment extensions.
'''
system = defs.get(system_spec['path'])
deploy_defaults = system_spec.get('deploy-defaults')
if system.get('arch') and system['arch'] != app.settings['arch']:
app.log(system, 'Skipping deployment for', system['arch'])
return None
sandbox.setup(system)
app.log(system, 'Extracting system artifact into', system['sandbox'])
with open(cache.get_cache(defs, system), 'r') as artifact:
call(['tar', 'x', '--directory', system['sandbox']],
stdin=artifact)
for subsystem_spec in system_spec.get('subsystems', []):
if deploy_defaults:
subsystem_spec = dict(deploy_defaults.items()
+ subsystem_spec.items())
deploy_system(defs, subsystem_spec, parent_location=system['sandbox'])
for name, deployment in system_spec.get('deploy', {}).iteritems():
method = os.path.basename(deployment['type'])
if deploy_defaults:
deployment = dict(deploy_defaults.items()
+ deployment.items())
do_deployment_manifest(system, deployment)
if parent_location:
deployment['location'] = os.path.join(
parent_location, deployment['location'].lstrip('/'))
try:
sandbox.run_extension(system, deployment, 'check', method)
except KeyError:
app.log(system, "Couldn't find a check extension for",
method)
for ext in system.get('configuration-extensions', []):
sandbox.run_extension(system, deployment, 'configure',
os.path.basename(ext))
os.chmod(system['sandbox'], 0o755)
sandbox.run_extension(system, deployment, 'write', method)
sandbox.remove(system)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
return cache.cache_key(defs, target)
component = defs.get(target)
if component.get('arch') and component['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'Starting assembly'):
sandbox.setup(component)
for system_spec in component.get('systems', []):
assemble_system_recursively(system_spec)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
sandbox.install(defs, component, subcomponent)
app.settings['counter'] += 1
if 'systems' not in component:
build(defs, component)
do_manifest(component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return cache.cache_key(target)
defs = Definitions()
this = defs.get(target)
if this.get('arch') and this['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', this['arch'])
return None
with app.timer(this, 'Starting assembly'):
sandbox.setup(this)
for it in this.get('systems', []):
system = defs.get(it)
assemble(system)
for subsystem in this.get('subsystems', []):
assemble(subsystem)
dependencies = this.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
contents = this.get('contents', [])
random.shuffle(contents)
for it in contents:
component = defs.get(it)
if component.get('build-mode') != 'bootstrap':
assemble(component)
sandbox.install(this, component)
build(this)
do_manifest(this)
cache.cache(this, full_root=this.get('kind', None) == "system")
sandbox.remove(this)
return cache.cache_key(this)
def main():
args = parse_args()
if args.list:
print("The available suites are:")
print('\n'.join([" - " + s.name for s in Suite.available]))
print("Groups:")
print('\n'.join([" - " + ', '.join(s.groups)
for s in Suite.available]))
sys.exit(0)
if config.MINISHELL_MAKE or args.make:
try:
subprocess.run(["make", "-C", config.MINISHELL_DIR], check=True)
except subprocess.CalledProcessError:
sys.exit(1)
if args.make:
sys.exit(0)
if os.path.exists(config.EXECUTABLES_PATH):
shutil.rmtree(config.EXECUTABLES_PATH)
os.mkdir(config.EXECUTABLES_PATH)
for cmd in config.AVAILABLE_COMMANDS:
shutil.copy(
distutils.spawn.find_executable(cmd), # FIXME search whole PATH
os.path.join(config.EXECUTABLES_PATH, cmd))
config.VERBOSE_LEVEL = args.verbose
if args.bonus:
config.BONUS = True
if args.no_bonus:
config.BONUS = False
Suite.setup(args.suites)
try:
Suite.run_all()
except KeyboardInterrupt:
sandbox.remove()
Suite.summarize()
Suite.save_log()
print("See", config.LOG_PATH, "for more information")
def _run_sandboxed(self, shell_path: str, shell_option: str) -> Captured:
""" run the command in a sandbox environment
capture the output (stdout and stderr)
capture the content of the watched files after the command is run
"""
sandbox.create()
if self.setup != "":
try:
setup_status = subprocess.run(self.setup,
shell=True,
cwd=config.SANDBOX_PATH,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
check=True)
except subprocess.CalledProcessError as e:
print("Error: `{}` setup command failed for `{}`\n\twith '{}'".
format(
self.setup, self.cmd, "no stderr"
if e.stdout is None else e.stdout.decode().strip()))
sys.exit(1)
process = subprocess.Popen(
[shell_path, shell_option, self.cmd],
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
cwd=config.SANDBOX_PATH,
env={
'PATH': config.PATH_VARIABLE,
'TERM': 'xterm-256color',
**self.exports,
},
)
if self.signal is not None:
time.sleep(0.2)
process.send_signal(self.signal)
else:
try:
process.wait(timeout=self.timeout)
except subprocess.TimeoutExpired:
return Captured.timeout()
try:
stdout, _ = process.communicate()
output = stdout.decode()
except UnicodeDecodeError:
output = "UNICODE ERROR: {}".format(process.stdout)
# capture watched files content
files_content = []
for file_name in self.files:
try:
with open(os.path.join(config.SANDBOX_PATH, file_name),
"rb") as f:
files_content.append(f.read().decode())
except FileNotFoundError as e:
files_content.append(None)
sandbox.remove()
if self.hook is not None:
output = self.hook(output)
return Captured(output, process.returncode, files_content)
