def seq_mux_map(desc, mapping):
desc, inner_configure = command.mux_map(desc, mapping)
def configure(command: list, parser: argparse.ArgumentParser):
# allow --dry-run to be present before selector and also have it appear in the help message
add_dry_run_argument(parser, "dry_run_outer")
inner_configure(command, parser)
return desc, configure
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_decrypt_file(keypath, keyout)
util.copy(certpath, certout)
keytab_command = command.mux_map(
"commands about keytabs granted by external sources", {
"import":
command.wrap("import and encrypt a keytab for a particular server",
import_keytab),
"rotate":
command.wrap(
"decrypt, rotate, and re-encrypt the keytab for a particular server",
rotate_keytab),
"delold":
command.wrap(
"decrypt, delete old entries from, and re-encrypt a keytab",
delold_keytab),
"list":
command.wrap("decrypt and list one or all of the stored keytabs",
list_keytabs),
"export":
command.wrap("decrypt and export the keytab for a particular server",
export_keytab),
})
https_command = command.mux_map(
"commands about HTTPS certs granted by external sources", {
"import":
command.wrap(
"import and encrypt a HTTPS keypair for a particular server",
for filename in inclusion).encode(),
cwd=d)
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=cddir).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=cddir)
util.writefile(os.path.join(cddir, "md5sum.txt"), md5s)
temp_iso = os.path.join(d, "temp.iso")
subprocess.check_call([
"xorriso", "-as", "mkisofs", "-quiet", "-o", temp_iso, "-r", "-J",
"-c", "boot.cat", "-b", "isolinux.bin", "-no-emul-boot",
"-boot-load-size", "4", "-boot-info-table", cddir
])
subprocess.check_call(["isohybrid", "-h", "64", "-s", "32", temp_iso])
util.copy(temp_iso, iso_image)
main_command = command.mux_map(
"commands about building installation ISOs", {
"gen":
command.wrap("generate ISO", gen_iso),
"passphrases":
command.wrap(
"decrypt a list of passphrases used by recently-generated ISOs",
list_passphrases),
})
cwd=d)
subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")])
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=os.path.join(d, "cd")).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=os.path.join(d, "cd"))
util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s)
subprocess.check_call([
"genisoimage", "-quiet", "-o", iso_image, "-r", "-J",
"-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b",
"isolinux.bin", "-c", "isolinux.cat",
os.path.join(d, "cd")
])
main_command = command.mux_map(
"commands about building installation ISOs", {
"regen-cdpack":
command.wrap("regenerate cdpack from upstream ISO", regen_cdpack),
"gen":
command.wrap("generate ISO", gen_iso),
"passphrases":
command.wrap(
"decrypt a list of passphrases used by recently-generated ISOs",
list_passphrases),
})
import infra
import keys
import seq
import deploy
import version
import virt
main_command = command.mux_map(
"invoke a top-level command", {
"iso": iso.main_command,
"config": configuration.main_command,
"authority": authority.main_command,
"keytab": keys.keytab_command,
"https": keys.https_command,
"setup": setup.main_command,
"query": query.main_command,
"verify": verify.main_command,
"access": access.main_command,
"etcdctl": access.etcdctl_command,
"kubectl": access.kubectl_command,
"foreach": access.foreach_command,
"infra": infra.main_command,
"seq": seq.main_command,
"deploy": deploy.main_command,
"virt": virt.main_command,
"version": version.main_command
})
if __name__ == "__main__":
sys.exit(command.main_invoke(main_command))
def get_keyurl_data(path):
config = configuration.Config.load_from_project()
keyserver_hostname = config.keyserver.hostname
url = "https://%s.%s:20557/%s" % (keyserver_hostname,
config.external_domain, path.lstrip("/"))
try:
with get_verified_keyserver_opener().open(url) as req:
if req.code != 200:
command.fail("request failed: %s" % req.read().decode())
return req.read().decode()
except urllib.error.HTTPError as e:
if e.code == 400:
command.fail("request failed: 400 " + e.msg +
" (possibly an auth error?)")
elif e.code == 404:
command.fail("path not found: 404 " + e.msg)
else:
raise e
def query_keyurl(path):
print(get_keyurl_data(path))
main_command = command.mux_map(
"commands about querying the state of a cluster", {
"keyurl":
command.wrap("request data from unprotected URLs on keyserver",
query_keyurl),
})
if not (node_kind == "node" or node_kind in valid_node_kinds):
command.fail("usage: spire foreach {node," +
",".join(valid_node_kinds) + "} command")
for node in config.nodes:
if node_kind == "node" or node.kind == node_kind:
ops.ssh("run command on @HOST", node, *params)
etcdctl_command = command.wrap("invoke commands through the etcdctl wrapper",
dispatch_etcdctl)
kubectl_command = command.wrap("invoke commands through the kubectl wrapper",
dispatch_kubectl)
foreach_command = setup.wrapop(
"invoke commands on every node (or every node of a given kind) in the cluster",
ssh_foreach)
main_command = command.mux_map(
"commands about establishing access to a cluster", {
"ssh":
command.wrap(
"request SSH access to the cluster and add it to the SSH agent",
access_ssh_with_add),
"ssh-fetch":
command.wrap(
"request SSH access to the cluster but do not register it with the agent",
access_ssh),
"update-known-hosts":
command.wrap(
"update ~/.ssh/known_hosts file with @ca-certificates directive",
update_known_hosts)
})
tokens[node.hostname] = (node.kind, node.ip, token)
print("host".center(16, "="), "kind".center(8, "="), "ip".center(14, "="),
"token".center(23, "="))
for key, (kind, ip, token) in sorted(tokens.items()):
print(key.rjust(16), kind.center(8),
str(ip).center(14), token.ljust(23))
print("host".center(16, "="), "kind".center(8, "="), "ip".center(14, "="),
"token".center(23, "="))
def infra_install_packages(ops: setup.Operations) -> None:
config = configuration.get_config()
for node in config.nodes:
ops.ssh("update apt repositories on @HOST", node, "apt-get", "update")
ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y")
main_command = command.mux_map(
"commands about maintaining the infrastructure of a cluster", {
"admit":
command.wrap("request a token to admit a node to the cluster",
infra_admit),
"admit-all":
command.wrap(
"request tokens to admit every non-supervisor node to the cluster",
infra_admit_all),
"install-packages":
setup.wrapop("install and update packages on a node",
infra_install_packages),
})
],
input="".join(
"%s\n" % filename
for filename in inclusion).encode(),
cwd=d)
subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")])
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=os.path.join(d, "cd")).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=os.path.join(d, "cd"))
util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s)
subprocess.check_call([
"genisoimage", "-quiet", "-o", iso_image, "-r", "-J",
"-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b",
"isolinux.bin", "-c", "isolinux.cat",
os.path.join(d, "cd")
])
main_command = command.mux_map(
"commands about building installation ISOs", {
"regen-cdpack":
command.wrap("regenerate cdpack from upstream ISO", regen_cdpack),
"gen":
command.wrap("generate ISO", gen_iso),
})
main_command = command.mux_map(
"commands about setting up a cluster", {
"keyserver":
wrapop("deploy keys and configuration for keyserver; start keyserver",
setup_keyserver),
"self-admit":
wrapop("admit the keyserver into the cluster during bootstrapping",
admit_keyserver),
"keygateway":
wrapop("deploy keytab and start keygateway", setup_keygateway),
"update-keygateway":
wrapop("update keytab and restart keygateway", update_keygateway),
"supervisor-ssh":
wrapop("configure supervisor SSH access", setup_supervisor_ssh),
"dns-bootstrap":
wrapop("switch cluster nodes into 'bootstrapped DNS' mode",
setup_dns_bootstrap),
"stop-dns-bootstrap":
wrapop("switch cluster nodes out of 'bootstrapped DNS' mode",
teardown_dns_bootstrap),
"bootstrap-registry":
wrapop(
"bring up the bootstrap container registry on the supervisor nodes",
setup_bootstrap_registry),
"prometheus":
wrapop("bring up the supervisor node prometheus instance",
setup_prometheus),
})
import command
import tempfile
import access
import configuration
import util
import os
def launch_spec(spec_name):
with tempfile.TemporaryDirectory() as d:
specfile = os.path.join(d, "spec.yaml")
util.writefile(specfile,
configuration.get_single_kube_spec(spec_name).encode())
access.call_kubectl(["apply", "-f", specfile], return_result=False)
main_command = command.mux_map(
"commands to deploy systems onto the kubernetes cluster", {
"flannel":
command.wrap("deploy the specifications to run flannel",
lambda: launch_spec("flannel.yaml")),
"dns-addon":
command.wrap("deploy the specifications to run the dns-addon",
lambda: launch_spec("dns-addon.yaml")),
})
lambda: iso.gen_iso(iso_path, authorized_key, "serial"))
with ops.context("networking", net_context()):
with ops.context("termination", TerminationContext()) as tc:
with ops.context("debug shell", DebugContext()):
ops.add_subcommand(lambda ops: auto_supervisor(
ops, tc, config.keyserver, iso_path))
for node in config.nodes:
if node == config.keyserver: continue
ops.add_subcommand(
lambda ops, n=node: auto_node(ops, tc, n, iso_path))
ops.add_subcommand(seq.sequence_cluster)
main_command = seq.seq_mux_map(
"commands to run local testing VMs", {
"net":
command.mux_map(
"commands to control the state of the local testing network", {
"up": command.wrap("bring up local testing network", net_up),
"down": command.wrap("bring down local testing network",
net_down),
}),
"auto":
seq.seq_mux_map(
"commands to perform large-scale operations automatically", {
"cluster":
seq.wrapseq("complete cluster installation", auto_cluster),
}),
})
import sys
import command
import configuration
import authority
import iso
import setup
import query
import verify
import access
import infra
main_command = command.mux_map(
"invoke a top-level command", {
"iso": iso.main_command,
"config": configuration.main_command,
"authority": authority.main_command,
"setup": setup.main_command,
"query": query.main_command,
"verify": verify.main_command,
"access": access.main_command,
"etcdctl": access.etcdctl_command,
"kubectl": access.kubectl_command,
"infra": infra.main_command,
})
if __name__ == "__main__":
sys.exit(command.main_invoke(main_command, sys.argv[1:]))
ops.add_operation("deploy or update dns-addon", deploy.launch_dns_addon)
ops.add_operation("verify that dns-addon is online", iterative_verifier(verify.check_dns_kubeinfo, 60.0))
ops.add_operation("verify that dns-addon is functioning", verify.check_dns_function)
ops.print_annotations("set up the dns-addon")
def sequence_addons(ops: setup.Operations) -> None:
ops.add_operation("deploy or update flannel", deploy.launch_flannel)
ops.add_operation("deploy or update dns-addon", deploy.launch_dns_addon)
ops.add_operation("verify that flannel is online", iterative_verifier(verify.check_flannel_kubeinfo, 60.0))
ops.add_operation("verify that dns-addon is online", iterative_verifier(verify.check_dns_kubeinfo, 60.0))
ops.add_operation("verify that flannel is functioning", verify.check_flannel_function)
ops.add_operation("verify that dns-addon is functioning", verify.check_dns_function)
ops.print_annotations("set up the dns-addon")
main_command = command.mux_map("commands about running large sequences of cluster bring-up automatically", {
"keysystem": setup.wrapop("set up and verify functionality of the keyserver and keygateway", sequence_keysystem),
"ssh": setup.wrapop("set up and verify ssh access to the supervisor node", sequence_ssh),
"supervisor": setup.wrapop("set up and verify functionality of entire supervisor node (keysystem + ssh)", sequence_supervisor),
"core": setup.wrapop("set up and verify core infrastructure operation", sequence_core),
"registry": setup.wrapop("set up and verify the bootstrap container registry", sequence_registry),
"flannel": setup.wrapop("set up and verify the flannel core service", sequence_flannel),
"dns-addon": setup.wrapop("set up and verify the dns-addon core service", sequence_dns_addon),
"addons": setup.wrapop("set up and verify the flannel and dns-addon core services", sequence_addons),
})
main_command = command.mux_map(
"commands about verifying the state of a cluster", {
"keystatics":
command.wrap(
"verify that keyserver static files are being served properly",
check_keystatics),
"keygateway":
command.wrap("verify that the keygateway has been properly started",
check_keygateway),
"online":
command.wrap(
"check whether a server (or all servers) is/are accepting SSH connections",
check_online),
"ssh-with-certs":
command.wrap("check if certificate-based SSH access works",
check_ssh_with_certs),
"supervisor-certs":
command.wrap(
"verify that certificates have been uploaded to the supervisor",
check_certs_on_supervisor),
"etcd":
command.wrap("verify that etcd is healthy and working",
check_etcd_health),
"kubernetes-init":
command.wrap("verify that kubernetes appears initialized",
check_kube_init),
"kubernetes":
command.wrap("verify that kubernetes appears healthy",
check_kube_health),
"aci-pull":
command.wrap(
"verify that aci pulling from the homeworld registry, and associated container execution, are functioning",
check_aci_pull),
"flannel":
command.wrap("verify that the flannel addon is functioning",
check_flannel),
"dns-addon":
command.wrap("verify that the DNS addon is functioning", check_dns),
})
main_command = command.mux_map(
"commands about verifying the state of a cluster", {
"keystatics":
command.wrap(
"verify that keyserver static files are being served properly",
check_keystatics),
"keygateway":
command.wrap("verify that the keygateway has been properly started",
check_keygateway),
"online":
command.wrap(
"check whether a server (or all servers) is/are accepting SSH connections",
check_online),
"ssh-with-certs":
command.wrap("check if certificate-based SSH access works",
check_ssh_with_certs),
"etcd":
command.wrap("verify that etcd is healthy and working",
check_etcd_health),
"kubernetes":
command.wrap("verify that kubernetes appears healthy",
check_kube_health),
"aci-pull":
command.wrap(
"verify that rkt can (or was able to) pull containers from the homeworld registry",
check_aci_pull),
"flannel-run":
command.wrap(
"verify that kubernetes has launched flannel successfully",
check_flannel_kubeinfo),
"flannel-ping":
command.wrap(
"verify that flannel-enabled containers on different hosts can talk",
check_flannel_function),
"dns-addon-run":
command.wrap(
"verify that kubernetes has launched dns-addon successfully",
check_dns_kubeinfo),
"dns-addon-query":
command.wrap(
"verify that the DNS addon is successfully handling requests",
check_dns_function),
})
import command
import tempfile
import access
import configuration
import util
import os
def launch_spec(spec_name):
with tempfile.TemporaryDirectory() as d:
specfile = os.path.join(d, "spec.yaml")
util.writefile(specfile, configuration.get_single_kube_spec(spec_name).encode())
access.call_kubectl(["apply", "-f", specfile], return_result=False)
def launch_flannel():
launch_spec("flannel.yaml")
def launch_dns_addon():
launch_spec("dns-addon.yaml")
main_command = command.mux_map("commands to deploy systems onto the kubernetes cluster", {
"flannel": command.wrap("deploy the specifications to run flannel", launch_flannel),
"dns-addon": command.wrap("deploy the specifications to run the dns-addon", launch_dns_addon),
})
main_command = command.mux_map(
"commands about cluster configuration", {
"populate":
command.wrap("initialize the cluster's setup.yaml with the template",
populate),
"edit":
command.wrap(
"open $EDITOR (defaults to nano) to edit the project's setup.yaml",
edit),
"gen-kube":
command.wrap("generate kubernetes specs for the base cluster",
gen_kube_specs),
"show":
command.mux_map(
"commands about showing different aspects of the configuration", {
"keyserver.yaml":
command.wrap("display the generated keyserver.yaml",
print_keyserver_yaml),
"keyclient.yaml":
command.wrap("display the specified variant of keyclient.yaml",
print_keyclient_yaml),
"cluster.conf":
command.wrap("display the generated cluster.conf",
print_cluster_conf),
"machine.list":
command.wrap("display the generated machine.list",
print_machine_list_file),
"kubeconfig":
command.wrap("display the generated local kubeconfig",
print_local_kubeconfig),
}),
})
pull_supervisor_key(
util.readfile(source_file).decode().strip().split("\n"))
etcdctl_command = command.wrap("invoke commands through the etcdctl wrapper",
dispatch_etcdctl)
kubectl_command = command.wrap("invoke commands through the kubectl wrapper",
dispatch_kubectl)
foreach_command = setup.wrapop(
"invoke commands on every node (or every node of a given kind) in the cluster",
ssh_foreach)
main_command = command.mux_map(
"commands about establishing access to a cluster", {
"ssh":
command.wrap(
"request SSH access to the cluster and add it to the SSH agent",
access_ssh_with_add),
"ssh-fetch":
command.wrap(
"request SSH access to the cluster but do not register it with the agent",
access_ssh),
"update-known-hosts":
command.wrap(
"update ~/.ssh/known_hosts file with @ca-certificates directive",
update_known_hosts),
"pull-supervisor-key":
command.wrap(
"update ~/.ssh/known_hosts file with the supervisor host keys, based on their known hashes",
pull_supervisor_key_from),
})
def iterate_keys(): # yields (name, contents) pairs
authorities = get_targz_path()
with tarfile.open(authorities, mode="r:gz") as tar:
for member in tar.getmembers():
if member.isreg():
with tar.extractfile(member) as f:
contents = f.read()
assert type(contents) == bytes
if member.name.startswith("./"):
yield member.name[2:], contents
else:
yield member.name, contents
def iterate_keys_decrypted(): # yields (name, contents) pairs
for name, contents in iterate_keys():
if name.endswith(".pub") or name.endswith(".pem"):
yield name, contents
else:
yield name_for_decrypted_file(
name), keycrypt.gpg_decrypt_in_memory(contents)
main_command = command.mux_map(
"commands about cluster authorities", {
"gen":
command.wrap("generate and encrypt authority keys and certs",
generate),
})
ikey = authority.get_decrypted_by_filename("./kubernetes.key")
icert = authority.get_pubkey_by_filename("./kubernetes.pem")
ikey64, icert64 = base64.b64encode(ikey), base64.b64encode(icert)
launch_spec(
"user-grant.yaml", {
"SERVER_KEY_BASE64": skey64.decode(),
"SERVER_CERT_BASE64": scert64.decode(),
"ISSUER_KEY_BASE64": ikey64.decode(),
"ISSUER_CERT_BASE64": icert64.decode(),
})
main_command = command.mux_map(
"commands to deploy systems onto the kubernetes cluster", {
"flannel":
command.wrap("deploy the specifications to run flannel",
launch_flannel),
"flannel-monitor":
command.wrap("deploy the specifications to run the flannel monitor",
launch_flannel_monitor),
"dns-addon":
command.wrap("deploy the specifications to run the dns-addon",
launch_dns_addon),
"dns-monitor":
command.wrap("deploy the specifications to run the dns monitor",
launch_dns_monitor),
"user-grant":
command.wrap("deploy the specifications to run the user grant website",
launch_user_grant),
})
