def wrapop(desc: str, f): def wrap_param_tx(params): config = configuration.Config.load_from_project() ops = Operations(config) return [ops, config] + params, ops.run_operations return command.wrap(desc, f, wrap_param_tx)
def wrapseq(desc: str, f): def wrap_param_tx(args): ops = setup.Operations() def invoke(): if args.dry_run or args.dry_run_outer: ops.print_annotations() else: ops.run_operations() return [ops] + args.params, invoke desc, inner_configure = command.wrap(desc, f, wrap_param_tx) def configure(command: list, parser: argparse.ArgumentParser): add_dry_run_argument(parser, "dry_run") inner_configure(command, parser) return desc, configure
cwd=d) subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")]) files_for_md5sum = subprocess.check_output( ["find", ".", "-follow", "-type", "f", "-print0"], cwd=os.path.join(d, "cd")).decode().split("\0") assert files_for_md5sum.pop() == "" md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum, cwd=os.path.join(d, "cd")) util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s) subprocess.check_call([ "genisoimage", "-quiet", "-o", iso_image, "-r", "-J", "-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b", "isolinux.bin", "-c", "isolinux.cat", os.path.join(d, "cd") ]) main_command = command.mux_map( "commands about building installation ISOs", { "regen-cdpack": command.wrap("regenerate cdpack from upstream ISO", regen_cdpack), "gen": command.wrap("generate ISO", gen_iso), "passphrases": command.wrap( "decrypt a list of passphrases used by recently-generated ISOs", list_passphrases), })
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) keys = hostkeys_by_fingerprint(node, fingerprints) with open(known_hosts, "a") as f: for key in keys: f.write("%s.%s %s\n" % (node.hostname, config.external_domain, key)) def pull_supervisor_key_from(source_file): pull_supervisor_key( util.readfile(source_file).decode().strip().split("\n")) etcdctl_command = command.wrap("invoke commands through the etcdctl wrapper", dispatch_etcdctl) kubectl_command = command.wrap("invoke commands through the kubectl wrapper", dispatch_kubectl) foreach_command = setup.wrapop( "invoke commands on every node (or every node of a given kind) in the cluster", ssh_foreach) main_command = command.mux_map( "commands about establishing access to a cluster", { "ssh": command.wrap( "request SSH access to the cluster and add it to the SSH agent", access_ssh_with_add), "ssh-fetch": command.wrap( "request SSH access to the cluster but do not register it with the agent", access_ssh),
import command import tempfile import access import configuration import util import os def launch_spec(spec_name): with tempfile.TemporaryDirectory() as d: specfile = os.path.join(d, "spec.yaml") util.writefile(specfile, configuration.get_single_kube_spec(spec_name).encode()) access.call_kubectl(["apply", "-f", specfile], return_result=False) def launch_flannel(): launch_spec("flannel.yaml") def launch_dns_addon(): launch_spec("dns-addon.yaml") main_command = command.mux_map("commands to deploy systems onto the kubernetes cluster", { "flannel": command.wrap("deploy the specifications to run flannel", launch_flannel), "dns-addon": command.wrap("deploy the specifications to run the dns-addon", launch_dns_addon), })
with tempfile.TemporaryDirectory() as d: certdir = os.path.join(d, "certdir") keyserver_yaml = os.path.join(d, "keyserver.yaml") util.writefile(keyserver_yaml, configuration.get_keyserver_yaml().encode()) os.mkdir(certdir) print("generating authorities...") try: subprocess.check_call(["keygen", keyserver_yaml, certdir, "supervisor-nodes"]) except FileNotFoundError as e: if e.filename == "keygen": command.fail("could not find keygen binary. is the homeworld-keyserver dependency installed?") else: raise e print("packing authorities...") subprocess.check_call(["tar", "-C", certdir, "-czf", authorities, "."]) subprocess.check_call(["shred", "--"] + os.listdir(certdir), cwd=certdir) def get_key_by_filename(keyname) -> bytes: authorities = get_targz_path() with tarfile.open(authorities, mode="r:gz") as tar: with tar.extractfile(keyname) as f: out = f.read() assert type(out) == bytes return out main_command = command.mux_map("commands about cluster authorities", { "gen": command.wrap("generate authorities keys and certs", generate), })
validator=lambda _, x: type(x) == list) vars = get_kube_spec_vars() for configname in clustered: templated = template.template("clustered/%s" % configname, vars) util.writefile(os.path.join(output_dir, configname), templated.encode()) def get_single_kube_spec(name: str) -> str: return template.template("clustered/%s" % name, get_kube_spec_vars()) main_command = command.mux_map( "commands about cluster configuration", { "populate": command.wrap("initialize the cluster's setup.yaml with the template", populate), "edit": command.wrap( "open $EDITOR (defaults to nano) to edit the project's setup.yaml", edit), "gen-kube": command.wrap("generate kubernetes specs for the base cluster", gen_kube_specs), "show": command.mux_map( "commands about showing different aspects of the configuration", { "keyserver.yaml": command.wrap("display the generated keyserver.yaml", print_keyserver_yaml), "keyclient.yaml": command.wrap("display the specified variant of keyclient.yaml",
"root@%s.%s" % (worker.hostname, config.external_domain), "--" ] + server_command) last_line = results.replace(b"\r\n", b"\n").replace(b"\0", b'').strip().split(b"\n")[-1] if not last_line.endswith(b"Address: 172.28.0.1"): command.fail("unexpected last line: %s" % repr(last_line.decode())) print("dns-addon seems to work!") main_command = command.mux_map( "commands about verifying the state of a cluster", { "keystatics": command.wrap( "verify that keyserver static files are being served properly", check_keystatics), "keygateway": command.wrap("verify that the keygateway has been properly started", check_keygateway), "online": command.wrap( "check whether a server (or all servers) is/are accepting SSH connections", check_online), "ssh-with-certs": command.wrap("check if certificate-based SSH access works", check_ssh_with_certs), "etcd": command.wrap("verify that etcd is healthy and working", check_etcd_health), "kubernetes":
def wrapop(desc: str, f): def wrap_param_tx(args): ops = Operations() return [ops] + args.params, ops.run_operations return command.wrap(desc, f, wrap_param_tx)
], input="".join( "%s\n" % filename for filename in inclusion).encode(), cwd=d) subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")]) files_for_md5sum = subprocess.check_output( ["find", ".", "-follow", "-type", "f", "-print0"], cwd=os.path.join(d, "cd")).decode().split("\0") assert files_for_md5sum.pop() == "" md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum, cwd=os.path.join(d, "cd")) util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s) subprocess.check_call([ "genisoimage", "-quiet", "-o", iso_image, "-r", "-J", "-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b", "isolinux.bin", "-c", "isolinux.cat", os.path.join(d, "cd") ]) main_command = command.mux_map( "commands about building installation ISOs", { "regen-cdpack": command.wrap("regenerate cdpack from upstream ISO", regen_cdpack), "gen": command.wrap("generate ISO", gen_iso), })
import command import tempfile import access import configuration import util import os def launch_spec(spec_name): with tempfile.TemporaryDirectory() as d: specfile = os.path.join(d, "spec.yaml") util.writefile(specfile, configuration.get_single_kube_spec(spec_name).encode()) access.call_kubectl(["apply", "-f", specfile], return_result=False) main_command = command.mux_map( "commands to deploy systems onto the kubernetes cluster", { "flannel": command.wrap("deploy the specifications to run flannel", lambda: launch_spec("flannel.yaml")), "dns-addon": command.wrap("deploy the specifications to run the dns-addon", lambda: launch_spec("dns-addon.yaml")), })
lambda: iso.gen_iso(iso_path, authorized_key, "serial")) with ops.context("networking", net_context()): with ops.context("termination", TerminationContext()) as tc: with ops.context("debug shell", DebugContext()): ops.add_subcommand(lambda ops: auto_supervisor( ops, tc, config.keyserver, iso_path)) for node in config.nodes: if node == config.keyserver: continue ops.add_subcommand( lambda ops, n=node: auto_node(ops, tc, n, iso_path)) ops.add_subcommand(seq.sequence_cluster) main_command = seq.seq_mux_map( "commands to run local testing VMs", { "net": command.mux_map( "commands to control the state of the local testing network", { "up": command.wrap("bring up local testing network", net_up), "down": command.wrap("bring down local testing network", net_down), }), "auto": seq.seq_mux_map( "commands to perform large-scale operations automatically", { "cluster": seq.wrapseq("complete cluster installation", auto_cluster), }), })
def get_keyurl_data(path): config = configuration.Config.load_from_project() keyserver_hostname = config.keyserver.hostname url = "https://%s.%s:20557/%s" % (keyserver_hostname, config.external_domain, path.lstrip("/")) try: with get_verified_keyserver_opener().open(url) as req: if req.code != 200: command.fail("request failed: %s" % req.read().decode()) return req.read().decode() except urllib.error.HTTPError as e: if e.code == 400: command.fail("request failed: 400 " + e.msg + " (possibly an auth error?)") elif e.code == 404: command.fail("path not found: 404 " + e.msg) else: raise e def query_keyurl(path): print(get_keyurl_data(path)) main_command = command.mux_map( "commands about querying the state of a cluster", { "keyurl": command.wrap("request data from unprotected URLs on keyserver", query_keyurl), })
import os import command from resources import get_resource def get_git_version(): return get_resource("GIT_VERSION").decode().rstrip() def display_version(): deb_version = get_resource("DEB_VERSION").decode().rstrip() print("Debian package version:", deb_version) git_version = get_git_version() print("Git commit hash:", git_version) main_command = command.wrap("display version info", display_version)
for filename in inclusion).encode(), cwd=d) files_for_md5sum = subprocess.check_output( ["find", ".", "-follow", "-type", "f", "-print0"], cwd=cddir).decode().split("\0") assert files_for_md5sum.pop() == "" md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum, cwd=cddir) util.writefile(os.path.join(cddir, "md5sum.txt"), md5s) temp_iso = os.path.join(d, "temp.iso") subprocess.check_call([ "xorriso", "-as", "mkisofs", "-quiet", "-o", temp_iso, "-r", "-J", "-c", "boot.cat", "-b", "isolinux.bin", "-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", cddir ]) subprocess.check_call(["isohybrid", "-h", "64", "-s", "32", temp_iso]) util.copy(temp_iso, iso_image) main_command = command.mux_map( "commands about building installation ISOs", { "gen": command.wrap("generate ISO", gen_iso), "passphrases": command.wrap( "decrypt a list of passphrases used by recently-generated ISOs", list_passphrases), })
def call_kubectl(params, return_result: bool): kubeconfig_data = configuration.get_local_kubeconfig() key_path, cert_path, ca_path = get_kube_cert_paths() if needs_rotate(cert_path): print("rotating kubernetes certs...") call_keyreq("kube-cert", key_path, cert_path, ca_path) with tempfile.TemporaryDirectory() as f: kubeconfig_path = os.path.join(f, "temp-kubeconfig") util.writefile(kubeconfig_path, kubeconfig_data.encode()) args = ["hyperkube", "kubectl", "--kubeconfig", kubeconfig_path] + list(params) if return_result: return subprocess.check_output(args) else: subprocess.check_call(args) def dispatch_kubectl(*params: str): call_kubectl(params, False) etcdctl_command = command.wrap("invoke commands through the etcdctl wrapper", dispatch_etcdctl) kubectl_command = command.wrap("invoke commands through the kubectl wrapper", dispatch_kubectl) main_command = command.mux_map("commands about establishing access to a cluster", { "ssh": command.wrap("request SSH access to the cluster and add it to the SSH agent", access_ssh_with_add), "ssh-fetch": command.wrap("request SSH access to the cluster but do not register it with the agent", access_ssh), "update-known-hosts": command.wrap("update ~/.ssh/known_hosts file with @ca-certificates directive", update_known_hosts) })
def export_https(name, keyout, certout): if name != setup.REGISTRY_HOSTNAME: command.fail("unexpected https host: %s" % name) keypath = os.path.join(configuration.get_project(), "https.%s.key.crypt" % name) certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name) keycrypt.gpg_decrypt_file(keypath, keyout) util.copy(certpath, certout) keytab_command = command.mux_map( "commands about keytabs granted by external sources", { "import": command.wrap("import and encrypt a keytab for a particular server", import_keytab), "rotate": command.wrap( "decrypt, rotate, and re-encrypt the keytab for a particular server", rotate_keytab), "delold": command.wrap( "decrypt, delete old entries from, and re-encrypt a keytab", delold_keytab), "list": command.wrap("decrypt and list one or all of the stored keytabs", list_keytabs), "export": command.wrap("decrypt and export the keytab for a particular server", export_keytab), })
if spec_replicas < 2: command.fail("not enough replicas requested by spec") if ready_replicas < spec_replicas - 1: # TODO: require precise results; not currently possible due to issues with DNS containers command.fail("not enough DNS replicas are ready") if float(pull_prometheus_query('avg(dns_lookup_internal_check)')) < 1: command.fail("dns lookup check failed") if float(pull_prometheus_query('time() - min(dns_lookup_recency)')) > 30: command.fail("dns lookup check is not recent enough") print("dns-addon seems to work!") main_command = command.mux_map( "commands about verifying the state of a cluster", { "keystatics": command.wrap( "verify that keyserver static files are being served properly", check_keystatics), "keygateway": command.wrap("verify that the keygateway has been properly started", check_keygateway), "online": command.wrap( "check whether a server (or all servers) is/are accepting SSH connections", check_online), "ssh-with-certs": command.wrap("check if certificate-based SSH access works", check_ssh_with_certs), "supervisor-certs": command.wrap( "verify that certificates have been uploaded to the supervisor", check_certs_on_supervisor),
tokens[node.hostname] = (node.kind, node.ip, token) print("host".center(16, "="), "kind".center(8, "="), "ip".center(14, "="), "token".center(23, "=")) for key, (kind, ip, token) in sorted(tokens.items()): print(key.rjust(16), kind.center(8), str(ip).center(14), token.ljust(23)) print("host".center(16, "="), "kind".center(8, "="), "ip".center(14, "="), "token".center(23, "=")) def infra_install_packages(ops: setup.Operations) -> None: config = configuration.get_config() for node in config.nodes: ops.ssh("update apt repositories on @HOST", node, "apt-get", "update") ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y") main_command = command.mux_map( "commands about maintaining the infrastructure of a cluster", { "admit": command.wrap("request a token to admit a node to the cluster", infra_admit), "admit-all": command.wrap( "request tokens to admit every non-supervisor node to the cluster", infra_admit_all), "install-packages": setup.wrapop("install and update packages on a node", infra_install_packages), })
import access import configuration import command import setup def infra_admit(server_principal: str) -> None: token = access.call_keyreq("bootstrap-token", server_principal, collect=True) print("Token granted for %s: '%s'" % (server_principal, token.decode().strip())) def infra_install_packages(ops: setup.Operations, config: configuration.Config) -> None: for node in config.nodes: ops.ssh("update apt repositories on @HOST", node, "apt-get", "update") ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y") if node.kind == "supervisor": ops.ssh("install supervisor packages on @HOST", node, "apt-get", "install", "-y", "homeworld-bootstrap-registry") else: ops.ssh("install standard packages on @HOST", node, "apt-get", "install", "-y", "homeworld-services") main_command = command.mux_map("commands about maintaining the infrastructure of a cluster", { "admit": command.wrap("request a token to admit a node to the cluster", infra_admit), "install-packages": setup.wrapop("install and update packages on a node", infra_install_packages), })
def iterate_keys(): # yields (name, contents) pairs authorities = get_targz_path() with tarfile.open(authorities, mode="r:gz") as tar: for member in tar.getmembers(): if member.isreg(): with tar.extractfile(member) as f: contents = f.read() assert type(contents) == bytes if member.name.startswith("./"): yield member.name[2:], contents else: yield member.name, contents def iterate_keys_decrypted(): # yields (name, contents) pairs for name, contents in iterate_keys(): if name.endswith(".pub") or name.endswith(".pem"): yield name, contents else: yield name_for_decrypted_file( name), keycrypt.gpg_decrypt_in_memory(contents) main_command = command.mux_map( "commands about cluster authorities", { "gen": command.wrap("generate and encrypt authority keys and certs", generate), })
launch_spec("flannel.yaml") def launch_flannel_monitor(): launch_spec("flannel-monitor.yaml") def launch_dns_addon(): launch_spec("dns-addon.yaml") def launch_dns_monitor(): launch_spec("dns-monitor.yaml") main_command = command.mux_map( "commands to deploy systems onto the kubernetes cluster", { "flannel": command.wrap("deploy the specifications to run flannel", launch_flannel), "flannel-monitor": command.wrap("deploy the specifications to run the flannel monitor", launch_flannel_monitor), "dns-addon": command.wrap("deploy the specifications to run the dns-addon", launch_dns_addon), "dns-monitor": command.wrap("deploy the specifications to run the dns monitor", launch_dns_monitor), })