async def cook(self, kitchen: Kitchen) -> None: # TODO(documentation): note this does not update users # acknowledge tracking kitchen.get_dependency_tracker() if self.password: password_hash: Optional[str] = crypt.crypt(self.password) else: password_hash = None pwd_entry = await kitchen.ut1a(GetPasswdEntry(self.user_name), GetPasswdEntry.Result) if pwd_entry: logger.warning( "Not updating existing os-user '%s' as it exists already and " "modifications could be dangerous in any case. Modification " "support may be implemented in the future.", self.user_name, ) else: # create the user fresh await linux_steps.create_linux_user( kitchen, self.user_name, password_hash, self.make_home, self.make_group, self.home, )
async def cook(self, k: Kitchen): for directory in self._make: stat = await k.ut1a(Stat(directory), Stat.Result) if stat is None: # doesn't exist, make it await k.ut0(MakeDirectory(directory, self.mode)) stat = await k.ut1a(Stat(directory), Stat.Result) if stat is None: raise RuntimeError("Directory vanished after creation!") if stat.dir: if (stat.user, stat.group) != (self.targ_user, self.targ_group): # need to chown await k.ut0( Chown(directory, self.targ_user, self.targ_group)) if stat.mode != self.mode: await k.ut0(Chmod(directory, self.mode)) else: raise RuntimeError("Already exists but not a dir: " + directory) # mark as tracked. k.get_dependency_tracker()
async def cook(self, kitchen: Kitchen) -> None: kitchen.get_dependency_tracker() await kitchen.ut1areq( DockerNetworkCreate( self.name, self.check_duplicate, self.internal, self.enable_ipv6, self.attachable, self.ingress, ), DockerNetworkCreate.Result, )
async def cook(self, kitchen: Kitchen) -> None: kitchen.get_dependency_tracker().ignore() changed = await kitchen.ut1( HasChangedInSousStore(self.purpose, self.watching)) if changed: result = await kitchen.ut1areq( SimpleExec(self.command, self.working_dir), SimpleExec.Result) if result.exit_code != 0: raise RuntimeError( f"exit code not 0 ({result.exit_code}), {result.stderr!r}")
async def load_and_transform(kitchen: Kitchen, meta: FridgeMetadata, fullpath: Path, sous: str) -> bytes: head = kitchen.head # TODO(perf) don't do this in async loop with fullpath.open("rb") as file: data = file.read() if meta == FridgeMetadata.FROZEN: # decrypt if head.secret_access is None: raise RuntimeError("Frozen file but no secret access enabled!") data = head.secret_access.decrypt_bytes(data) elif meta == FridgeMetadata.TEMPLATE: # pass through Jinja2 try: env = Environment(loader=DictLoader({str(fullpath): data.decode()}), autoescape=False) template = env.get_template(str(fullpath)) proxies = kitchen.get_dependency_tracker().get_j2_var_proxies( head.variables[sous]) data = template.render(proxies).encode() except Exception as e: raise RuntimeError(f"Error templating: {fullpath}") from e # try: # return jinja2.utils.concat( # template.root_render_func(template.new_context(proxies)) # ) # except Exception: # template.environment.handle_exception() return data
async def cook(self, kitchen: Kitchen) -> None: # this is a one-off task assuming everything works kitchen.get_dependency_tracker() if self.packages: update = await self._apt_command(kitchen, ["apt-get", "-yq", "update"]) if update.exit_code != 0: raise RuntimeError( f"apt update failed with err {update.exit_code}: {update.stderr!r}" ) install_args = ["apt-get", "-yq", "install"] install_args += list(self.packages) install = await self._apt_command(kitchen, install_args) if install.exit_code != 0: raise RuntimeError( f"apt install failed with err {install.exit_code}:" f" {install.stderr!r}" )
async def cook(self, kitchen: Kitchen): dt = kitchen.get_dependency_tracker() await exec_no_fails(kitchen, [self.interpreter, "-m", "venv", self.dir], "/") install_args = [] for name, flags in self.install: if "-r" in flags: install_args.append("-r") await depend_remote_file(name, kitchen) elif "dir" in flags or "git" in flags: # TODO(perf, dedup): custom dynamic dependency types; git # dependencies and sha256_dir dependencies. dt.ignore() install_args.append(name) await exec_no_fails(kitchen, [self.dir + "/bin/pip", "install"] + install_args, "/")
async def depend_remote_file(path: str, kitchen: Kitchen) -> None: sha256 = await kitchen.ut1(HashFile(path)) kitchen.get_dependency_tracker().register_remote_file(path, sha256)
async def cook(self, kitchen: Kitchen) -> None: kitchen.get_dependency_tracker()
async def cook(self, kitchen: Kitchen) -> None: kitchen.get_dependency_tracker() await kitchen.ut1areq(DockerVolumeCreate(self.name), DockerVolumeCreate.Result)
async def cook(self, kitchen: Kitchen) -> None: kitchen.get_dependency_tracker() await kitchen.ut1areq(DockerImagePull(self.repository, self.tag), DockerImagePull.Result)
async def cook(self, kitchen: Kitchen) -> None: kitchen.get_dependency_tracker() await kitchen.ut1areq(DockerContainerRun(self.image, self.command), DockerContainerRun.Result)
async def cook(self, kitchen: Kitchen): # mark as tracked. kitchen.get_dependency_tracker()
async def cook(self, k: Kitchen) -> None: # no non-arg dependencies k.get_dependency_tracker() stat = await k.ut1a(Stat(self.dest_dir), Stat.Result) if stat is None: # doesn't exist; git init it await exec_no_fails(k, ["git", "init", self.dest_dir], "/") stat = await k.ut1a(Stat(self.dest_dir), Stat.Result) if stat is None: raise RuntimeError("Directory vanished after creation!") if not stat.dir: raise RuntimeError("Already exists but not a dir: " + self.dest_dir) # add the remote, removing it first to ensure it's what we want # don't care if removing fails await k.ut1areq( SimpleExec(["git", "remote", "remove", "scone"], self.dest_dir), SimpleExec.Result, ) await exec_no_fails(k, ["git", "remote", "add", "scone", self.repo_src], self.dest_dir) # fetch the latest from the remote await exec_no_fails(k, ["git", "fetch", "scone"], self.dest_dir) # figure out what ref we want to use # TODO(performance): fetch only this ref? ref = self.ref or f"scone/{self.branch}" # switch to that ref await exec_no_fails(k, ["git", "switch", "--detach", ref], self.dest_dir) # if we use submodules if self.submodules: await exec_no_fails( k, ["git", "submodule", "update", "--init", "--recursive"], self.dest_dir, ) for expected in self.expect: expected_path_str = str(Path(self.dest_dir, expected)) # TODO(performance, low): parallelise these stat = await k.ut1a(Stat(expected_path_str), Stat.Result) if not stat: raise RuntimeError( f"expected {expected_path_str} to exist but it did not") if stat.dir and not expected.endswith("/"): raise RuntimeError( f"expected {expected_path_str} to exist as a file but it is a dir" ) if not stat.dir and expected.endswith("/"): raise RuntimeError( f"expected {expected_path_str} to exist as a dir but it is a file" )
async def cli_async() -> int: dep_cache = None try: args = sys.argv[1:] parser = ArgumentParser(description="Cook!") parser.add_argument("hostspec", type=str, help="Sous or group name") parser.add_argument( "--yes", "-y", action="store_true", default=False, help="Don't prompt for confirmation", ) argp = parser.parse_args(args) eprint("Loading head…") cdir = Path(os.getcwd()) while not Path(cdir, "scone.head.toml").exists(): cdir = cdir.parent if len(cdir.parts) <= 1: eprint("Don't appear to be in a head. STOP.") return 1 head = Head.open(str(cdir)) eprint(head.debug_info()) hosts = set() if argp.hostspec in head.souss: hosts.add(argp.hostspec) elif argp.hostspec in head.groups: for sous in head.groups[argp.hostspec]: hosts.add(sous) else: eprint(f"Unrecognised sous or group: '{argp.hostspec}'") sys.exit(1) eprint(f"Selected the following souss: {', '.join(hosts)}") eprint("Preparing recipes…") prepare = Preparation(head) start_ts = time.monotonic() prepare.prepare_all() del prepare end_ts = time.monotonic() eprint(f"Preparation completed in {end_ts - start_ts:.3f} s.") # eprint(f"{len(order)} courses planned.") dot_emitter.emit_dot(head.dag, Path(cdir, "dag.0.dot")) dep_cache = await DependencyCache.open( os.path.join(head.directory, "depcache.sqlite3")) # eprint("Checking dependency cache…") # start_ts = time.monotonic() # depchecks = await run_dep_checks(head, dep_cache, order) # end_ts = time.monotonic() # eprint(f"Checking finished in {end_ts - start_ts:.3f} s.") # TODO show counts # # for epoch, items in enumerate(order): # print(f"----- Course {epoch} -----") # # for item in items: # if isinstance(item, Recipe): # state = depchecks[item].label.name # print(f" > recipe ({state}) {item}") # elif isinstance(item, tuple): # kind, ident, extra = item # print(f" - we now have {kind} {ident} {dict(extra)}") eprint("Ready to cook? [y/N]: ", end="") if argp.yes: eprint("y (due to --yes)") else: if not input().lower().startswith("y"): eprint("Stopping.") return 101 kitchen = Kitchen(head, dep_cache) # for epoch, epoch_items in enumerate(order): # print(f"Cooking Course {epoch} of {len(order)}") # await kitchen.run_epoch( # epoch_items, depchecks, concurrency_limit_per_host=8 # ) # # for sous in hosts: TODO this is not definitely safe # await dep_cache.sweep_old(sous) try: await kitchen.cook_all() finally: dot_emitter.emit_dot(head.dag, Path(cdir, "dag.9.dot")) return 0 finally: Pools.get().shutdown() if dep_cache: await dep_cache.db.close()