async def cli_async() -> int:
args = sys.argv[1:]
parser = ArgumentParser(description="Compose a menu!")
subs = parser.add_subparsers()
supermarket = subs.add_parser("supermarket",
help="generate a [[supermarket]] dish")
supermarket.add_argument("url", help="HTTPS URL to download")
supermarket.add_argument("-a", "--as", help="Alternative filename")
supermarket.set_defaults(func=supermarket_cli)
argp = parser.parse_args(args)
if not hasattr(argp, "func"):
parser.print_help()
return 127
cdir = Path(os.getcwd())
while not Path(cdir, "scone.head.toml").exists():
cdir = cdir.parent
if len(cdir.parts) <= 1:
eprint("Don't appear to be in a head. STOP.")
sys.exit(1)
with open(join(cdir, "scone.head.toml")) as head_toml:
head_data = toml.load(head_toml)
return await argp.func(argp, head_data, cdir)
def open(directory: str):
with open(path.join(directory, "scone.head.toml")) as head_toml:
head_data = toml.load(head_toml)
secret_access: Optional[SecretAccess] = None
if "freezer" in head_data and "restaurant_id" in head_data["freezer"]:
secret_access = SecretAccess(head_data["freezer"]["restaurant_id"])
secret_access.get_existing()
if not secret_access.key:
eprint("Failed to load freezer secret.")
sys.exit(12)
recipe_module_roots = head_data.get("recipe_roots", ["scone.default.recipes"])
# load available recipes
recipe_loader: ClassLoader[Recipe] = ClassLoader(Recipe, recipe_name_getter)
for recipe_root in recipe_module_roots:
recipe_loader.add_package_root(recipe_root)
sous = head_data.get("sous", dict())
groups = head_data.get("group", dict())
groups["all"] = list(sous.keys())
pools = Pools()
head = Head(directory, recipe_loader, sous, groups, secret_access, pools)
head._load_variables()
head._load_menus()
return head
def _try_manual_entry(self) -> Optional[bytes]:
eprint("Manual entry required. Enter password for this restaurant: ", end="")
key = URLSafeBase64Encoder.decode(input().encode())
if len(key) != SecretBox.KEY_SIZE:
eprint("Wrong size!")
return None
else:
return key
def _try_dbus_auth(self, restaurant_identifier: str) -> Optional[bytes]:
eprint("Trying D-Bus Secret Service")
try:
with secretstorage.dbus_init() as connection:
collection = secretstorage.get_default_collection(connection)
attributes = {
"application": "Scone",
"restaurant": restaurant_identifier,
}
items = list(collection.search_items(attributes))
if items:
eprint("Found secret sauce for this Restaurant, unlocking…")
items[0].unlock()
return URLSafeBase64Encoder.decode(items[0].get_secret())
else:
eprint("Did not find secret sauce for this Restaurant.")
eprint("Enter it and I will try and store it...")
secret = self._try_manual_entry()
if secret is not None:
collection.create_item(
f"scone({restaurant_identifier}): secret sauce",
attributes,
URLSafeBase64Encoder.encode(secret),
)
return secret
return None
except EOFError: # XXX what happens with no D-Bus
return None
async def _cooking_worker(self):
dag = self.head.dag
while True:
if self._sleeper_slots <= 0 and self._cookable.empty():
self._sleeper_slots -= 1
self._cookable.put_nowait(None)
break
self._sleeper_slots -= 1
try:
next_job = await self._cookable.get()
finally:
self._sleeper_slots += 1
if next_job is None:
continue
if isinstance(next_job, Recipe):
meta = dag.recipe_meta[next_job]
# TODO try to deduplicate
meta.state = RecipeState.BEING_COOKED
current_recipe.set(next_job)
eprint(f"cooking {next_job}")
self._dependency_trackers[next_job] = DependencyTracker(
DependencyBook(), dag, next_job)
try:
await next_job.cook(self)
except Exception as e:
meta.state = RecipeState.FAILED
raise RuntimeError(f"Recipe {next_job} failed!") from e
eprint(f"cooked {next_job}")
# TODO cook
# TODO store depbook
await self._store_dependency(next_job)
meta.state = RecipeState.COOKED
elif isinstance(next_job, Resource):
eprint(f"have {next_job}")
pass
for edge in dag.edges[next_job]:
logger.debug("updating edge: %s → %s", next_job, edge)
if isinstance(edge, Recipe):
rec_meta = dag.recipe_meta[edge]
rec_meta.incoming_uncompleted -= 1
logger.debug("has %d incoming",
rec_meta.incoming_uncompleted)
if (rec_meta.incoming_uncompleted == 0
and rec_meta.state == RecipeState.PENDING):
rec_meta.state = RecipeState.COOKABLE
self._cookable.put_nowait(edge)
elif isinstance(edge, Resource):
res_meta = dag.resource_meta[edge]
res_meta.incoming_uncompleted -= 1
logger.debug("has %d incoming",
res_meta.incoming_uncompleted)
if res_meta.incoming_uncompleted == 0 and not res_meta.completed:
res_meta.completed = True
self._cookable.put_nowait(edge)
async def supermarket_cli(argp, head_data: dict, head_dir: Path) -> int:
eprint("Want to download", argp.url)
r = requests.get(argp.url, stream=True)
with tempfile.NamedTemporaryFile(delete=False) as tfp:
filename = tfp.name
for chunk in r.iter_content(4 * 1024 * 1024):
tfp.write(chunk)
eprint("Hashing", filename)
real_sha256 = sha256_file(filename)
note = f"""
Scone Supermarket
This file corresponds to {argp.url}
Downloaded by michelin.
""".strip()
target_path = Path(head_dir, ".scone-cache", "supermarket", real_sha256)
target_path.parent.mkdir(parents=True, exist_ok=True)
shutil.move(filename, str(target_path))
with open(str(target_path) + ".txt", "w") as fout:
# leave a note so we can find out what this is if we need to.
fout.write(note)
print("[[supermarket]]")
print(f'url = "{argp.url}"')
print(f'sha256 = "{real_sha256}"')
print("dest = ")
print("#owner = bob")
print("#group = laura")
print('#mode = "ug=rw,o=r"')
return 0
def generate_new(self):
eprint("Generating a new freezer key...")
self.key = nacl.utils.random(SecretBox.KEY_SIZE)
key_b64 = URLSafeBase64Encoder.encode(self.key)
eprint("Your new key is: " + key_b64.decode())
eprint("Pretty please store it in a safe place!")
if not self.restaurant_identifier:
eprint("No RI; not saving to SS")
return
eprint("Attempting to save it to the secret service...")
eprint("(save it yourself anyway!)")
with secretstorage.dbus_init() as connection:
collection = secretstorage.get_default_collection(connection)
attributes = {
"application": "Scone",
"restaurant": self.restaurant_identifier,
}
items = list(collection.search_items(attributes))
if items:
eprint(
"Found secret sauce for this Restaurant already!"
" Will not overwrite."
)
else:
eprint("Storing secret sauce for this Restaurant...")
collection.create_item(
f"scone({self.restaurant_identifier}): secret sauce",
attributes,
key_b64,
)
eprint("OK!")
def cli() -> None:
args = sys.argv[1:]
if len(args) < 1:
eprint("Not enough arguments.")
eprint("Usage: scone-freezer <subcommand>:")
eprint(" freezefile <file> [small files only for now!]")
eprint(" thawfile <file>")
eprint(" freezevar <key> (value as 1 line in stdin)")
eprint(" thawvar <key>")
eprint(" genkey")
eprint(" test")
sys.exit(127)
cdir = Path(os.getcwd())
while not Path(cdir, "scone.head.toml").exists():
cdir = cdir.parent
if len(cdir.parts) <= 1:
eprint("Don't appear to be in a head. STOP.")
sys.exit(1)
with open(join(cdir, "scone.head.toml")) as head_toml:
head_data = toml.load(head_toml)
if "freezer" in head_data and "restaurant_id" in head_data["freezer"]:
restaurant_id = head_data["freezer"]["restaurant_id"]
else:
eprint("Tip: Set a freezer.restaurant_id in your scone.head.toml")
eprint(" to enable the ability to store your secret in the secret service.")
restaurant_id = None
secret_access = SecretAccess(restaurant_id)
if args[0] == "genkey":
assert len(args) == 1
secret_access.generate_new()
elif args[0] == "test":
secret_access.get_existing()
if secret_access.key:
eprint("Great! Key found.")
else:
eprint("Oh no! Key not found.")
elif args[0] == "freezefile":
secret_access.get_existing()
if not secret_access.key:
eprint("No key found!")
sys.exit(12)
assert len(args) >= 2
filepaths = [Path(p) for p in args[1:]]
ec = 0
for path in filepaths:
if not path.exists():
eprint(f"Can't freeze: no such file '{path}'")
sys.exit(10)
for path in filepaths:
eprint(f"Freezing {path}")
if not path.is_file():
eprint(f"Can't freeze (skipping): not a regular file '{path}'")
ec = 5
# slurping here for simplicity;
file_bytes = path.read_bytes()
enc_bytes = secret_access.encrypt_bytes(file_bytes)
dest_path = Path(str(path) + ".frozen")
dest_path.write_bytes(enc_bytes)
sys.exit(ec)
elif args[0] == "thawfile":
secret_access.get_existing()
if not secret_access.key:
eprint("No key found!")
sys.exit(12)
assert len(args) >= 2
filepaths = [Path(p) for p in args[1:]]
ec = 0
for path in filepaths:
if not path.exists():
eprint(f"Can't thaw: no such file '{path}'")
sys.exit(10)
for path in filepaths:
eprint(f"Thawing {path}")
if not path.is_file():
eprint(f"Can't thaw (skipping): not a regular file '{path}'")
ec = 5
continue
pathstr = str(path)
if not pathstr.endswith(".frozen"):
eprint(f"Can't thaw (skipping): not .frozen '{path}'")
continue
# slurping here for simplicity;
file_bytes = path.read_bytes()
dec_bytes = secret_access.decrypt_bytes(file_bytes)
dest_path = Path(str(pathstr[: -len(".frozen")]))
dest_path.write_bytes(dec_bytes)
sys.exit(ec)
elif args[0] == "freezevar":
assert len(args) == 2
secret_access.get_existing()
if not secret_access.key:
eprint("No key found!")
sys.exit(12)
key = args[1]
eprint("Enter value to freeze: ", end="", flush=True)
value = input()
enc_b64 = secret_access.encrypt_bytes(
value.encode(), encoder=URLSafeBase64Encoder
).decode()
n = 78
str_contents = "\n".join(
[" " + enc_b64[i : i + n] for i in range(0, len(enc_b64), n)]
)
print(f'{key} = """')
print(str_contents)
print('"""')
elif args[0] == "thawvar":
assert len(args) == 1
secret_access.get_existing()
if not secret_access.key:
eprint("No key found!")
sys.exit(12)
eprint("Enter base64 to thaw (whitespace removed painlessly) then EOF (^D):")
value = re.sub(r"\s", "", sys.stdin.read())
dec_str = secret_access.decrypt_bytes(
value.encode(), encoder=URLSafeBase64Encoder
).decode()
print(dec_str)
async def cli_async() -> int:
dep_cache = None
try:
args = sys.argv[1:]
parser = ArgumentParser(description="Cook!")
parser.add_argument("hostspec", type=str, help="Sous or group name")
parser.add_argument(
"--yes",
"-y",
action="store_true",
default=False,
help="Don't prompt for confirmation",
)
argp = parser.parse_args(args)
eprint("Loading head…")
cdir = Path(os.getcwd())
while not Path(cdir, "scone.head.toml").exists():
cdir = cdir.parent
if len(cdir.parts) <= 1:
eprint("Don't appear to be in a head. STOP.")
return 1
head = Head.open(str(cdir))
eprint(head.debug_info())
hosts = set()
if argp.hostspec in head.souss:
hosts.add(argp.hostspec)
elif argp.hostspec in head.groups:
for sous in head.groups[argp.hostspec]:
hosts.add(sous)
else:
eprint(f"Unrecognised sous or group: '{argp.hostspec}'")
sys.exit(1)
eprint(f"Selected the following souss: {', '.join(hosts)}")
eprint("Preparing recipes…")
prepare = Preparation(head)
start_ts = time.monotonic()
prepare.prepare_all()
del prepare
end_ts = time.monotonic()
eprint(f"Preparation completed in {end_ts - start_ts:.3f} s.")
# eprint(f"{len(order)} courses planned.")
dot_emitter.emit_dot(head.dag, Path(cdir, "dag.0.dot"))
dep_cache = await DependencyCache.open(
os.path.join(head.directory, "depcache.sqlite3"))
# eprint("Checking dependency cache…")
# start_ts = time.monotonic()
# depchecks = await run_dep_checks(head, dep_cache, order)
# end_ts = time.monotonic()
# eprint(f"Checking finished in {end_ts - start_ts:.3f} s.") # TODO show counts
#
# for epoch, items in enumerate(order):
# print(f"----- Course {epoch} -----")
#
# for item in items:
# if isinstance(item, Recipe):
# state = depchecks[item].label.name
# print(f" > recipe ({state}) {item}")
# elif isinstance(item, tuple):
# kind, ident, extra = item
# print(f" - we now have {kind} {ident} {dict(extra)}")
eprint("Ready to cook? [y/N]: ", end="")
if argp.yes:
eprint("y (due to --yes)")
else:
if not input().lower().startswith("y"):
eprint("Stopping.")
return 101
kitchen = Kitchen(head, dep_cache)
# for epoch, epoch_items in enumerate(order):
# print(f"Cooking Course {epoch} of {len(order)}")
# await kitchen.run_epoch(
# epoch_items, depchecks, concurrency_limit_per_host=8
# )
#
# for sous in hosts: TODO this is not definitely safe
# await dep_cache.sweep_old(sous)
try:
await kitchen.cook_all()
finally:
dot_emitter.emit_dot(head.dag, Path(cdir, "dag.9.dot"))
return 0
finally:
Pools.get().shutdown()
if dep_cache:
await dep_cache.db.close()