async def cook(self, kitchen: Kitchen) -> None:
# TODO(documentation): note this does not update users
# acknowledge tracking
kitchen.get_dependency_tracker()
if self.password:
password_hash: Optional[str] = crypt.crypt(self.password)
else:
password_hash = None
pwd_entry = await kitchen.ut1a(GetPasswdEntry(self.user_name),
GetPasswdEntry.Result)
if pwd_entry:
logger.warning(
"Not updating existing os-user '%s' as it exists already and "
"modifications could be dangerous in any case. Modification "
"support may be implemented in the future.",
self.user_name,
)
else:
# create the user fresh
await linux_steps.create_linux_user(
kitchen,
self.user_name,
password_hash,
self.make_home,
self.make_group,
self.home,
)
async def cook(self, k: Kitchen):
for directory in self._make:
stat = await k.ut1a(Stat(directory), Stat.Result)
if stat is None:
# doesn't exist, make it
await k.ut0(MakeDirectory(directory, self.mode))
stat = await k.ut1a(Stat(directory), Stat.Result)
if stat is None:
raise RuntimeError("Directory vanished after creation!")
if stat.dir:
if (stat.user, stat.group) != (self.targ_user,
self.targ_group):
# need to chown
await k.ut0(
Chown(directory, self.targ_user, self.targ_group))
if stat.mode != self.mode:
await k.ut0(Chmod(directory, self.mode))
else:
raise RuntimeError("Already exists but not a dir: " +
directory)
# mark as tracked.
k.get_dependency_tracker()
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker()
await kitchen.ut1areq(
DockerNetworkCreate(
self.name,
self.check_duplicate,
self.internal,
self.enable_ipv6,
self.attachable,
self.ingress,
),
DockerNetworkCreate.Result,
)
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker().ignore()
changed = await kitchen.ut1(
HasChangedInSousStore(self.purpose, self.watching))
if changed:
result = await kitchen.ut1areq(
SimpleExec(self.command, self.working_dir), SimpleExec.Result)
if result.exit_code != 0:
raise RuntimeError(
f"exit code not 0 ({result.exit_code}), {result.stderr!r}")
async def load_and_transform(kitchen: Kitchen, meta: FridgeMetadata,
fullpath: Path, sous: str) -> bytes:
head = kitchen.head
# TODO(perf) don't do this in async loop
with fullpath.open("rb") as file:
data = file.read()
if meta == FridgeMetadata.FROZEN:
# decrypt
if head.secret_access is None:
raise RuntimeError("Frozen file but no secret access enabled!")
data = head.secret_access.decrypt_bytes(data)
elif meta == FridgeMetadata.TEMPLATE:
# pass through Jinja2
try:
env = Environment(loader=DictLoader({str(fullpath):
data.decode()}),
autoescape=False)
template = env.get_template(str(fullpath))
proxies = kitchen.get_dependency_tracker().get_j2_var_proxies(
head.variables[sous])
data = template.render(proxies).encode()
except Exception as e:
raise RuntimeError(f"Error templating: {fullpath}") from e
# try:
# return jinja2.utils.concat(
# template.root_render_func(template.new_context(proxies))
# )
# except Exception:
# template.environment.handle_exception()
return data
async def cook(self, kitchen: Kitchen) -> None:
# this is a one-off task assuming everything works
kitchen.get_dependency_tracker()
if self.packages:
update = await self._apt_command(kitchen, ["apt-get", "-yq", "update"])
if update.exit_code != 0:
raise RuntimeError(
f"apt update failed with err {update.exit_code}: {update.stderr!r}"
)
install_args = ["apt-get", "-yq", "install"]
install_args += list(self.packages)
install = await self._apt_command(kitchen, install_args)
if install.exit_code != 0:
raise RuntimeError(
f"apt install failed with err {install.exit_code}:"
f" {install.stderr!r}"
)
async def cook(self, kitchen: Kitchen):
dt = kitchen.get_dependency_tracker()
await exec_no_fails(kitchen,
[self.interpreter, "-m", "venv", self.dir], "/")
install_args = []
for name, flags in self.install:
if "-r" in flags:
install_args.append("-r")
await depend_remote_file(name, kitchen)
elif "dir" in flags or "git" in flags:
# TODO(perf, dedup): custom dynamic dependency types; git
# dependencies and sha256_dir dependencies.
dt.ignore()
install_args.append(name)
await exec_no_fails(kitchen,
[self.dir + "/bin/pip", "install"] + install_args,
"/")
async def depend_remote_file(path: str, kitchen: Kitchen) -> None:
sha256 = await kitchen.ut1(HashFile(path))
kitchen.get_dependency_tracker().register_remote_file(path, sha256)
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker()
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker()
await kitchen.ut1areq(DockerVolumeCreate(self.name),
DockerVolumeCreate.Result)
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker()
await kitchen.ut1areq(DockerImagePull(self.repository, self.tag),
DockerImagePull.Result)
async def cook(self, kitchen: Kitchen) -> None:
kitchen.get_dependency_tracker()
await kitchen.ut1areq(DockerContainerRun(self.image, self.command),
DockerContainerRun.Result)
async def cook(self, kitchen: Kitchen):
# mark as tracked.
kitchen.get_dependency_tracker()
async def cook(self, k: Kitchen) -> None:
# no non-arg dependencies
k.get_dependency_tracker()
stat = await k.ut1a(Stat(self.dest_dir), Stat.Result)
if stat is None:
# doesn't exist; git init it
await exec_no_fails(k, ["git", "init", self.dest_dir], "/")
stat = await k.ut1a(Stat(self.dest_dir), Stat.Result)
if stat is None:
raise RuntimeError("Directory vanished after creation!")
if not stat.dir:
raise RuntimeError("Already exists but not a dir: " +
self.dest_dir)
# add the remote, removing it first to ensure it's what we want
# don't care if removing fails
await k.ut1areq(
SimpleExec(["git", "remote", "remove", "scone"], self.dest_dir),
SimpleExec.Result,
)
await exec_no_fails(k,
["git", "remote", "add", "scone", self.repo_src],
self.dest_dir)
# fetch the latest from the remote
await exec_no_fails(k, ["git", "fetch", "scone"], self.dest_dir)
# figure out what ref we want to use
# TODO(performance): fetch only this ref?
ref = self.ref or f"scone/{self.branch}"
# switch to that ref
await exec_no_fails(k, ["git", "switch", "--detach", ref],
self.dest_dir)
# if we use submodules
if self.submodules:
await exec_no_fails(
k,
["git", "submodule", "update", "--init", "--recursive"],
self.dest_dir,
)
for expected in self.expect:
expected_path_str = str(Path(self.dest_dir, expected))
# TODO(performance, low): parallelise these
stat = await k.ut1a(Stat(expected_path_str), Stat.Result)
if not stat:
raise RuntimeError(
f"expected {expected_path_str} to exist but it did not")
if stat.dir and not expected.endswith("/"):
raise RuntimeError(
f"expected {expected_path_str} to exist as a file but it is a dir"
)
if not stat.dir and expected.endswith("/"):
raise RuntimeError(
f"expected {expected_path_str} to exist as a dir but it is a file"
)
async def cli_async() -> int:
dep_cache = None
try:
args = sys.argv[1:]
parser = ArgumentParser(description="Cook!")
parser.add_argument("hostspec", type=str, help="Sous or group name")
parser.add_argument(
"--yes",
"-y",
action="store_true",
default=False,
help="Don't prompt for confirmation",
)
argp = parser.parse_args(args)
eprint("Loading head…")
cdir = Path(os.getcwd())
while not Path(cdir, "scone.head.toml").exists():
cdir = cdir.parent
if len(cdir.parts) <= 1:
eprint("Don't appear to be in a head. STOP.")
return 1
head = Head.open(str(cdir))
eprint(head.debug_info())
hosts = set()
if argp.hostspec in head.souss:
hosts.add(argp.hostspec)
elif argp.hostspec in head.groups:
for sous in head.groups[argp.hostspec]:
hosts.add(sous)
else:
eprint(f"Unrecognised sous or group: '{argp.hostspec}'")
sys.exit(1)
eprint(f"Selected the following souss: {', '.join(hosts)}")
eprint("Preparing recipes…")
prepare = Preparation(head)
start_ts = time.monotonic()
prepare.prepare_all()
del prepare
end_ts = time.monotonic()
eprint(f"Preparation completed in {end_ts - start_ts:.3f} s.")
# eprint(f"{len(order)} courses planned.")
dot_emitter.emit_dot(head.dag, Path(cdir, "dag.0.dot"))
dep_cache = await DependencyCache.open(
os.path.join(head.directory, "depcache.sqlite3"))
# eprint("Checking dependency cache…")
# start_ts = time.monotonic()
# depchecks = await run_dep_checks(head, dep_cache, order)
# end_ts = time.monotonic()
# eprint(f"Checking finished in {end_ts - start_ts:.3f} s.") # TODO show counts
#
# for epoch, items in enumerate(order):
# print(f"----- Course {epoch} -----")
#
# for item in items:
# if isinstance(item, Recipe):
# state = depchecks[item].label.name
# print(f" > recipe ({state}) {item}")
# elif isinstance(item, tuple):
# kind, ident, extra = item
# print(f" - we now have {kind} {ident} {dict(extra)}")
eprint("Ready to cook? [y/N]: ", end="")
if argp.yes:
eprint("y (due to --yes)")
else:
if not input().lower().startswith("y"):
eprint("Stopping.")
return 101
kitchen = Kitchen(head, dep_cache)
# for epoch, epoch_items in enumerate(order):
# print(f"Cooking Course {epoch} of {len(order)}")
# await kitchen.run_epoch(
# epoch_items, depchecks, concurrency_limit_per_host=8
# )
#
# for sous in hosts: TODO this is not definitely safe
# await dep_cache.sweep_old(sous)
try:
await kitchen.cook_all()
finally:
dot_emitter.emit_dot(head.dag, Path(cdir, "dag.9.dot"))
return 0
finally:
Pools.get().shutdown()
if dep_cache:
await dep_cache.db.close()