def deploy(target): '''Deploy systems and subsystems recursively''' defs = Definitions() deployment = target if type(target) is dict else defs.get(target) with app.timer(deployment, 'Starting deployment'): for system in deployment.get('systems', []): deploy(system) for subsystem in system.get('subsystems', []): deploy(subsystem) system = defs.get(deployment['path']) if system.get('arch') and system['arch'] != app.settings['arch']: app.log(target, 'Skipping deployment for', system['arch']) return None sandbox.setup(system) for name, deployment in deployment.get('deploy', {}).iteritems(): method = os.path.basename(deployment['type']) sandbox.run_extension(system, deployment, 'check', method) app.log(system, "Extracting system artifact") with open(cache.get_cache(system), "r") as artifact: call(['tar', 'x', '--directory', system['sandbox']], stdin=artifact) for ext in system.get('configuration-extensions', []): sandbox.run_extension(system, deployment, 'configure', os.path.basename(ext)) os.chmod(system['sandbox'], 0o755) sandbox.run_extension(system, deployment, 'write', method) sandbox.remove(system)
def assemble(defs, target): '''Assemble dependencies and contents recursively until target exists.''' if cache.get_cache(defs, target): # needed for artifact splitting load_manifest(defs, target) return cache.cache_key(defs, target) random.seed(datetime.datetime.now()) component = defs.get(target) if component.get('arch') and component['arch'] != app.config['arch']: app.log(target, 'Skipping assembly for', component.get('arch')) return None def assemble_system_recursively(system): assemble(defs, system['path']) for subsystem in system.get('subsystems', []): assemble_system_recursively(subsystem) with app.timer(component, 'assembly'): sandbox.setup(component) systems = component.get('systems', []) random.shuffle(systems) for system in systems: assemble_system_recursively(system) dependencies = component.get('build-depends', []) random.shuffle(dependencies) for it in dependencies: dependency = defs.get(it) assemble(defs, dependency) sandbox.install(defs, component, dependency) contents = component.get('contents', []) random.shuffle(contents) for it in contents: subcomponent = defs.get(it) if subcomponent.get('build-mode') != 'bootstrap': assemble(defs, subcomponent) splits = None if component.get('kind') == 'system': splits = subcomponent.get('artifacts') sandbox.install(defs, component, subcomponent, splits) app.config['counter'] += 1 if 'systems' not in component: with app.timer(component, 'build'): build(defs, component) with app.timer(component, 'artifact creation'): do_manifest(defs, component) cache.cache(defs, component, full_root=component.get('kind') == "system") sandbox.remove(component) return cache.cache_key(defs, component)
def assemble(defs, target): '''Assemble dependencies and contents recursively until target exists.''' component = defs.get(target) if get_cache(defs, component) or get_remote(defs, component): return cache_key(defs, component) random.seed(datetime.datetime.now()) if component.get('arch') and component['arch'] != app.config['arch']: app.log(target, 'Skipping assembly for', component.get('arch')) return None sandbox.setup(component) systems = component.get('systems', []) random.shuffle(systems) for system in systems: assemble(defs, system['path']) for subsystem in system.get('subsystems', []): assemble(defs, subsystem) dependencies = component.get('build-depends', []) for it in dependencies: preinstall(defs, component, it) contents = component.get('contents', []) random.shuffle(contents) for it in contents: subcomponent = defs.get(it) if subcomponent.get('build-mode', 'staging') != 'bootstrap': preinstall(defs, component, subcomponent) if 'systems' not in component: if is_building(defs, component): import time time.sleep(10) raise Exception app.config['counter'] += 1 if not get_cache(defs, component): with app.timer(component, 'build of %s' % component['cache']): with claim(defs, component): build(defs, component) with app.timer(component, 'artifact creation'): do_manifest(component) cache(defs, component) sandbox.remove(component) return cache_key(defs, component)
def deploy_system(defs, system_spec, parent_location=''): '''Deploy a system and subsystems recursively. Takes a system spec (i.e. an entry in the "systems" list in a cluster definition), and optionally a path to a parent system tree. If `parent_location` is given then the `location` given in the cluster definition for the subsystem is appended to `parent_location`, with the result being used as the location for the deployment extensions. ''' system = defs.get(system_spec['path']) deploy_defaults = system_spec.get('deploy-defaults') if system.get('arch') and system['arch'] != app.settings['arch']: app.log(system, 'Skipping deployment for', system['arch']) return None sandbox.setup(system) app.log(system, 'Extracting system artifact into', system['sandbox']) with open(cache.get_cache(defs, system), 'r') as artifact: call(['tar', 'x', '--directory', system['sandbox']], stdin=artifact) for subsystem_spec in system_spec.get('subsystems', []): if deploy_defaults: subsystem_spec = dict(deploy_defaults.items() + subsystem_spec.items()) deploy_system(defs, subsystem_spec, parent_location=system['sandbox']) for name, deployment in system_spec.get('deploy', {}).iteritems(): method = os.path.basename(deployment['type']) if deploy_defaults: deployment = dict(deploy_defaults.items() + deployment.items()) do_deployment_manifest(system, deployment) if parent_location: deployment['location'] = os.path.join( parent_location, deployment['location'].lstrip('/')) try: sandbox.run_extension(system, deployment, 'check', method) except KeyError: app.log(system, "Couldn't find a check extension for", method) for ext in system.get('configuration-extensions', []): sandbox.run_extension(system, deployment, 'configure', os.path.basename(ext)) os.chmod(system['sandbox'], 0o755) sandbox.run_extension(system, deployment, 'write', method) sandbox.remove(system)
def assemble(defs, target): '''Assemble dependencies and contents recursively until target exists.''' if cache.get_cache(defs, target): return cache.cache_key(defs, target) component = defs.get(target) if component.get('arch') and component['arch'] != app.settings['arch']: app.log(target, 'Skipping assembly for', component.get('arch')) return None def assemble_system_recursively(system): assemble(defs, system['path']) for subsystem in system.get('subsystems', []): assemble_system_recursively(subsystem) with app.timer(component, 'Starting assembly'): sandbox.setup(component) for system_spec in component.get('systems', []): assemble_system_recursively(system_spec) dependencies = component.get('build-depends', []) random.shuffle(dependencies) for it in dependencies: dependency = defs.get(it) assemble(defs, dependency) sandbox.install(defs, component, dependency) contents = component.get('contents', []) random.shuffle(contents) for it in contents: subcomponent = defs.get(it) if subcomponent.get('build-mode') != 'bootstrap': assemble(defs, subcomponent) sandbox.install(defs, component, subcomponent) app.settings['counter'] += 1 if 'systems' not in component: build(defs, component) do_manifest(component) cache.cache(defs, component, full_root=component.get('kind') == "system") sandbox.remove(component) return cache.cache_key(defs, component)
def assemble(target): '''Assemble dependencies and contents recursively until target exists.''' if cache.get_cache(target): return cache.cache_key(target) defs = Definitions() this = defs.get(target) if this.get('arch') and this['arch'] != app.settings['arch']: app.log(target, 'Skipping assembly for', this['arch']) return None with app.timer(this, 'Starting assembly'): sandbox.setup(this) for it in this.get('systems', []): system = defs.get(it) assemble(system) for subsystem in this.get('subsystems', []): assemble(subsystem) dependencies = this.get('build-depends', []) random.shuffle(dependencies) for it in dependencies: dependency = defs.get(it) assemble(dependency) sandbox.install(this, dependency) contents = this.get('contents', []) random.shuffle(contents) for it in contents: component = defs.get(it) if component.get('build-mode') != 'bootstrap': assemble(component) sandbox.install(this, component) build(this) do_manifest(this) cache.cache(this, full_root=this.get('kind', None) == "system") sandbox.remove(this) return cache.cache_key(this)
def main(): args = parse_args() if args.list: print("The available suites are:") print('\n'.join([" - " + s.name for s in Suite.available])) print("Groups:") print('\n'.join([" - " + ', '.join(s.groups) for s in Suite.available])) sys.exit(0) if config.MINISHELL_MAKE or args.make: try: subprocess.run(["make", "-C", config.MINISHELL_DIR], check=True) except subprocess.CalledProcessError: sys.exit(1) if args.make: sys.exit(0) if os.path.exists(config.EXECUTABLES_PATH): shutil.rmtree(config.EXECUTABLES_PATH) os.mkdir(config.EXECUTABLES_PATH) for cmd in config.AVAILABLE_COMMANDS: shutil.copy( distutils.spawn.find_executable(cmd), # FIXME search whole PATH os.path.join(config.EXECUTABLES_PATH, cmd)) config.VERBOSE_LEVEL = args.verbose if args.bonus: config.BONUS = True if args.no_bonus: config.BONUS = False Suite.setup(args.suites) try: Suite.run_all() except KeyboardInterrupt: sandbox.remove() Suite.summarize() Suite.save_log() print("See", config.LOG_PATH, "for more information")
def _run_sandboxed(self, shell_path: str, shell_option: str) -> Captured: """ run the command in a sandbox environment capture the output (stdout and stderr) capture the content of the watched files after the command is run """ sandbox.create() if self.setup != "": try: setup_status = subprocess.run(self.setup, shell=True, cwd=config.SANDBOX_PATH, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=True) except subprocess.CalledProcessError as e: print("Error: `{}` setup command failed for `{}`\n\twith '{}'". format( self.setup, self.cmd, "no stderr" if e.stdout is None else e.stdout.decode().strip())) sys.exit(1) process = subprocess.Popen( [shell_path, shell_option, self.cmd], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, cwd=config.SANDBOX_PATH, env={ 'PATH': config.PATH_VARIABLE, 'TERM': 'xterm-256color', **self.exports, }, ) if self.signal is not None: time.sleep(0.2) process.send_signal(self.signal) else: try: process.wait(timeout=self.timeout) except subprocess.TimeoutExpired: return Captured.timeout() try: stdout, _ = process.communicate() output = stdout.decode() except UnicodeDecodeError: output = "UNICODE ERROR: {}".format(process.stdout) # capture watched files content files_content = [] for file_name in self.files: try: with open(os.path.join(config.SANDBOX_PATH, file_name), "rb") as f: files_content.append(f.read().decode()) except FileNotFoundError as e: files_content.append(None) sandbox.remove() if self.hook is not None: output = self.hook(output) return Captured(output, process.returncode, files_content)