def wrapop(desc: str, f):
def wrap_param_tx(params):
config = configuration.Config.load_from_project()
ops = Operations(config)
return [ops, config] + params, ops.run_operations
return command.wrap(desc, f, wrap_param_tx)
def wrapseq(desc: str, f):
def wrap_param_tx(args):
ops = setup.Operations()
def invoke():
if args.dry_run or args.dry_run_outer:
ops.print_annotations()
else:
ops.run_operations()
return [ops] + args.params, invoke
desc, inner_configure = command.wrap(desc, f, wrap_param_tx)
def configure(command: list, parser: argparse.ArgumentParser):
add_dry_run_argument(parser, "dry_run")
inner_configure(command, parser)
return desc, configure
cwd=d)
subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")])
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=os.path.join(d, "cd")).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=os.path.join(d, "cd"))
util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s)
subprocess.check_call([
"genisoimage", "-quiet", "-o", iso_image, "-r", "-J",
"-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b",
"isolinux.bin", "-c", "isolinux.cat",
os.path.join(d, "cd")
])
main_command = command.mux_map(
"commands about building installation ISOs", {
"regen-cdpack":
command.wrap("regenerate cdpack from upstream ISO", regen_cdpack),
"gen":
command.wrap("generate ISO", gen_iso),
"passphrases":
command.wrap(
"decrypt a list of passphrases used by recently-generated ISOs",
list_passphrases),
})
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
keys = hostkeys_by_fingerprint(node, fingerprints)
with open(known_hosts, "a") as f:
for key in keys:
f.write("%s.%s %s\n" %
(node.hostname, config.external_domain, key))
def pull_supervisor_key_from(source_file):
pull_supervisor_key(
util.readfile(source_file).decode().strip().split("\n"))
etcdctl_command = command.wrap("invoke commands through the etcdctl wrapper",
dispatch_etcdctl)
kubectl_command = command.wrap("invoke commands through the kubectl wrapper",
dispatch_kubectl)
foreach_command = setup.wrapop(
"invoke commands on every node (or every node of a given kind) in the cluster",
ssh_foreach)
main_command = command.mux_map(
"commands about establishing access to a cluster", {
"ssh":
command.wrap(
"request SSH access to the cluster and add it to the SSH agent",
access_ssh_with_add),
"ssh-fetch":
command.wrap(
"request SSH access to the cluster but do not register it with the agent",
access_ssh),
import command
import tempfile
import access
import configuration
import util
import os
def launch_spec(spec_name):
with tempfile.TemporaryDirectory() as d:
specfile = os.path.join(d, "spec.yaml")
util.writefile(specfile, configuration.get_single_kube_spec(spec_name).encode())
access.call_kubectl(["apply", "-f", specfile], return_result=False)
def launch_flannel():
launch_spec("flannel.yaml")
def launch_dns_addon():
launch_spec("dns-addon.yaml")
main_command = command.mux_map("commands to deploy systems onto the kubernetes cluster", {
"flannel": command.wrap("deploy the specifications to run flannel", launch_flannel),
"dns-addon": command.wrap("deploy the specifications to run the dns-addon", launch_dns_addon),
})
with tempfile.TemporaryDirectory() as d:
certdir = os.path.join(d, "certdir")
keyserver_yaml = os.path.join(d, "keyserver.yaml")
util.writefile(keyserver_yaml, configuration.get_keyserver_yaml().encode())
os.mkdir(certdir)
print("generating authorities...")
try:
subprocess.check_call(["keygen", keyserver_yaml, certdir, "supervisor-nodes"])
except FileNotFoundError as e:
if e.filename == "keygen":
command.fail("could not find keygen binary. is the homeworld-keyserver dependency installed?")
else:
raise e
print("packing authorities...")
subprocess.check_call(["tar", "-C", certdir, "-czf", authorities, "."])
subprocess.check_call(["shred", "--"] + os.listdir(certdir), cwd=certdir)
def get_key_by_filename(keyname) -> bytes:
authorities = get_targz_path()
with tarfile.open(authorities, mode="r:gz") as tar:
with tar.extractfile(keyname) as f:
out = f.read()
assert type(out) == bytes
return out
main_command = command.mux_map("commands about cluster authorities", {
"gen": command.wrap("generate authorities keys and certs", generate),
})
validator=lambda _, x: type(x) == list)
vars = get_kube_spec_vars()
for configname in clustered:
templated = template.template("clustered/%s" % configname, vars)
util.writefile(os.path.join(output_dir, configname),
templated.encode())
def get_single_kube_spec(name: str) -> str:
return template.template("clustered/%s" % name, get_kube_spec_vars())
main_command = command.mux_map(
"commands about cluster configuration", {
"populate":
command.wrap("initialize the cluster's setup.yaml with the template",
populate),
"edit":
command.wrap(
"open $EDITOR (defaults to nano) to edit the project's setup.yaml",
edit),
"gen-kube":
command.wrap("generate kubernetes specs for the base cluster",
gen_kube_specs),
"show":
command.mux_map(
"commands about showing different aspects of the configuration", {
"keyserver.yaml":
command.wrap("display the generated keyserver.yaml",
print_keyserver_yaml),
"keyclient.yaml":
command.wrap("display the specified variant of keyclient.yaml",
"root@%s.%s" % (worker.hostname, config.external_domain), "--"
] + server_command)
last_line = results.replace(b"\r\n",
b"\n").replace(b"\0",
b'').strip().split(b"\n")[-1]
if not last_line.endswith(b"Address: 172.28.0.1"):
command.fail("unexpected last line: %s" % repr(last_line.decode()))
print("dns-addon seems to work!")
main_command = command.mux_map(
"commands about verifying the state of a cluster", {
"keystatics":
command.wrap(
"verify that keyserver static files are being served properly",
check_keystatics),
"keygateway":
command.wrap("verify that the keygateway has been properly started",
check_keygateway),
"online":
command.wrap(
"check whether a server (or all servers) is/are accepting SSH connections",
check_online),
"ssh-with-certs":
command.wrap("check if certificate-based SSH access works",
check_ssh_with_certs),
"etcd":
command.wrap("verify that etcd is healthy and working",
check_etcd_health),
"kubernetes":
def wrapop(desc: str, f):
def wrap_param_tx(args):
ops = Operations()
return [ops] + args.params, ops.run_operations
return command.wrap(desc, f, wrap_param_tx)
],
input="".join(
"%s\n" % filename
for filename in inclusion).encode(),
cwd=d)
subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")])
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=os.path.join(d, "cd")).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=os.path.join(d, "cd"))
util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s)
subprocess.check_call([
"genisoimage", "-quiet", "-o", iso_image, "-r", "-J",
"-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b",
"isolinux.bin", "-c", "isolinux.cat",
os.path.join(d, "cd")
])
main_command = command.mux_map(
"commands about building installation ISOs", {
"regen-cdpack":
command.wrap("regenerate cdpack from upstream ISO", regen_cdpack),
"gen":
command.wrap("generate ISO", gen_iso),
})
import command
import tempfile
import access
import configuration
import util
import os
def launch_spec(spec_name):
with tempfile.TemporaryDirectory() as d:
specfile = os.path.join(d, "spec.yaml")
util.writefile(specfile,
configuration.get_single_kube_spec(spec_name).encode())
access.call_kubectl(["apply", "-f", specfile], return_result=False)
main_command = command.mux_map(
"commands to deploy systems onto the kubernetes cluster", {
"flannel":
command.wrap("deploy the specifications to run flannel",
lambda: launch_spec("flannel.yaml")),
"dns-addon":
command.wrap("deploy the specifications to run the dns-addon",
lambda: launch_spec("dns-addon.yaml")),
})
lambda: iso.gen_iso(iso_path, authorized_key, "serial"))
with ops.context("networking", net_context()):
with ops.context("termination", TerminationContext()) as tc:
with ops.context("debug shell", DebugContext()):
ops.add_subcommand(lambda ops: auto_supervisor(
ops, tc, config.keyserver, iso_path))
for node in config.nodes:
if node == config.keyserver: continue
ops.add_subcommand(
lambda ops, n=node: auto_node(ops, tc, n, iso_path))
ops.add_subcommand(seq.sequence_cluster)
main_command = seq.seq_mux_map(
"commands to run local testing VMs", {
"net":
command.mux_map(
"commands to control the state of the local testing network", {
"up": command.wrap("bring up local testing network", net_up),
"down": command.wrap("bring down local testing network",
net_down),
}),
"auto":
seq.seq_mux_map(
"commands to perform large-scale operations automatically", {
"cluster":
seq.wrapseq("complete cluster installation", auto_cluster),
}),
})
def get_keyurl_data(path):
config = configuration.Config.load_from_project()
keyserver_hostname = config.keyserver.hostname
url = "https://%s.%s:20557/%s" % (keyserver_hostname,
config.external_domain, path.lstrip("/"))
try:
with get_verified_keyserver_opener().open(url) as req:
if req.code != 200:
command.fail("request failed: %s" % req.read().decode())
return req.read().decode()
except urllib.error.HTTPError as e:
if e.code == 400:
command.fail("request failed: 400 " + e.msg +
" (possibly an auth error?)")
elif e.code == 404:
command.fail("path not found: 404 " + e.msg)
else:
raise e
def query_keyurl(path):
print(get_keyurl_data(path))
main_command = command.mux_map(
"commands about querying the state of a cluster", {
"keyurl":
command.wrap("request data from unprotected URLs on keyserver",
query_keyurl),
})
import os
import command
from resources import get_resource
def get_git_version():
return get_resource("GIT_VERSION").decode().rstrip()
def display_version():
deb_version = get_resource("DEB_VERSION").decode().rstrip()
print("Debian package version:", deb_version)
git_version = get_git_version()
print("Git commit hash:", git_version)
main_command = command.wrap("display version info", display_version)
for filename in inclusion).encode(),
cwd=d)
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=cddir).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=cddir)
util.writefile(os.path.join(cddir, "md5sum.txt"), md5s)
temp_iso = os.path.join(d, "temp.iso")
subprocess.check_call([
"xorriso", "-as", "mkisofs", "-quiet", "-o", temp_iso, "-r", "-J",
"-c", "boot.cat", "-b", "isolinux.bin", "-no-emul-boot",
"-boot-load-size", "4", "-boot-info-table", cddir
])
subprocess.check_call(["isohybrid", "-h", "64", "-s", "32", temp_iso])
util.copy(temp_iso, iso_image)
main_command = command.mux_map(
"commands about building installation ISOs", {
"gen":
command.wrap("generate ISO", gen_iso),
"passphrases":
command.wrap(
"decrypt a list of passphrases used by recently-generated ISOs",
list_passphrases),
})
def call_kubectl(params, return_result: bool):
kubeconfig_data = configuration.get_local_kubeconfig()
key_path, cert_path, ca_path = get_kube_cert_paths()
if needs_rotate(cert_path):
print("rotating kubernetes certs...")
call_keyreq("kube-cert", key_path, cert_path, ca_path)
with tempfile.TemporaryDirectory() as f:
kubeconfig_path = os.path.join(f, "temp-kubeconfig")
util.writefile(kubeconfig_path, kubeconfig_data.encode())
args = ["hyperkube", "kubectl", "--kubeconfig", kubeconfig_path] + list(params)
if return_result:
return subprocess.check_output(args)
else:
subprocess.check_call(args)
def dispatch_kubectl(*params: str):
call_kubectl(params, False)
etcdctl_command = command.wrap("invoke commands through the etcdctl wrapper", dispatch_etcdctl)
kubectl_command = command.wrap("invoke commands through the kubectl wrapper", dispatch_kubectl)
main_command = command.mux_map("commands about establishing access to a cluster", {
"ssh": command.wrap("request SSH access to the cluster and add it to the SSH agent", access_ssh_with_add),
"ssh-fetch": command.wrap("request SSH access to the cluster but do not register it with the agent", access_ssh),
"update-known-hosts": command.wrap("update ~/.ssh/known_hosts file with @ca-certificates directive", update_known_hosts)
})
def export_https(name, keyout, certout):
if name != setup.REGISTRY_HOSTNAME:
command.fail("unexpected https host: %s" % name)
keypath = os.path.join(configuration.get_project(),
"https.%s.key.crypt" % name)
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_decrypt_file(keypath, keyout)
util.copy(certpath, certout)
keytab_command = command.mux_map(
"commands about keytabs granted by external sources", {
"import":
command.wrap("import and encrypt a keytab for a particular server",
import_keytab),
"rotate":
command.wrap(
"decrypt, rotate, and re-encrypt the keytab for a particular server",
rotate_keytab),
"delold":
command.wrap(
"decrypt, delete old entries from, and re-encrypt a keytab",
delold_keytab),
"list":
command.wrap("decrypt and list one or all of the stored keytabs",
list_keytabs),
"export":
command.wrap("decrypt and export the keytab for a particular server",
export_keytab),
})
if spec_replicas < 2:
command.fail("not enough replicas requested by spec")
if ready_replicas < spec_replicas - 1: # TODO: require precise results; not currently possible due to issues with DNS containers
command.fail("not enough DNS replicas are ready")
if float(pull_prometheus_query('avg(dns_lookup_internal_check)')) < 1:
command.fail("dns lookup check failed")
if float(pull_prometheus_query('time() - min(dns_lookup_recency)')) > 30:
command.fail("dns lookup check is not recent enough")
print("dns-addon seems to work!")
main_command = command.mux_map(
"commands about verifying the state of a cluster", {
"keystatics":
command.wrap(
"verify that keyserver static files are being served properly",
check_keystatics),
"keygateway":
command.wrap("verify that the keygateway has been properly started",
check_keygateway),
"online":
command.wrap(
"check whether a server (or all servers) is/are accepting SSH connections",
check_online),
"ssh-with-certs":
command.wrap("check if certificate-based SSH access works",
check_ssh_with_certs),
"supervisor-certs":
command.wrap(
"verify that certificates have been uploaded to the supervisor",
check_certs_on_supervisor),
tokens[node.hostname] = (node.kind, node.ip, token)
print("host".center(16, "="), "kind".center(8, "="), "ip".center(14, "="),
"token".center(23, "="))
for key, (kind, ip, token) in sorted(tokens.items()):
print(key.rjust(16), kind.center(8),
str(ip).center(14), token.ljust(23))
print("host".center(16, "="), "kind".center(8, "="), "ip".center(14, "="),
"token".center(23, "="))
def infra_install_packages(ops: setup.Operations) -> None:
config = configuration.get_config()
for node in config.nodes:
ops.ssh("update apt repositories on @HOST", node, "apt-get", "update")
ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y")
main_command = command.mux_map(
"commands about maintaining the infrastructure of a cluster", {
"admit":
command.wrap("request a token to admit a node to the cluster",
infra_admit),
"admit-all":
command.wrap(
"request tokens to admit every non-supervisor node to the cluster",
infra_admit_all),
"install-packages":
setup.wrapop("install and update packages on a node",
infra_install_packages),
})
import access
import configuration
import command
import setup
def infra_admit(server_principal: str) -> None:
token = access.call_keyreq("bootstrap-token", server_principal, collect=True)
print("Token granted for %s: '%s'" % (server_principal, token.decode().strip()))
def infra_install_packages(ops: setup.Operations, config: configuration.Config) -> None:
for node in config.nodes:
ops.ssh("update apt repositories on @HOST", node, "apt-get", "update")
ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y")
if node.kind == "supervisor":
ops.ssh("install supervisor packages on @HOST", node, "apt-get", "install", "-y",
"homeworld-bootstrap-registry")
else:
ops.ssh("install standard packages on @HOST", node, "apt-get", "install", "-y",
"homeworld-services")
main_command = command.mux_map("commands about maintaining the infrastructure of a cluster", {
"admit": command.wrap("request a token to admit a node to the cluster", infra_admit),
"install-packages": setup.wrapop("install and update packages on a node", infra_install_packages),
})
def iterate_keys(): # yields (name, contents) pairs
authorities = get_targz_path()
with tarfile.open(authorities, mode="r:gz") as tar:
for member in tar.getmembers():
if member.isreg():
with tar.extractfile(member) as f:
contents = f.read()
assert type(contents) == bytes
if member.name.startswith("./"):
yield member.name[2:], contents
else:
yield member.name, contents
def iterate_keys_decrypted(): # yields (name, contents) pairs
for name, contents in iterate_keys():
if name.endswith(".pub") or name.endswith(".pem"):
yield name, contents
else:
yield name_for_decrypted_file(
name), keycrypt.gpg_decrypt_in_memory(contents)
main_command = command.mux_map(
"commands about cluster authorities", {
"gen":
command.wrap("generate and encrypt authority keys and certs",
generate),
})
launch_spec("flannel.yaml")
def launch_flannel_monitor():
launch_spec("flannel-monitor.yaml")
def launch_dns_addon():
launch_spec("dns-addon.yaml")
def launch_dns_monitor():
launch_spec("dns-monitor.yaml")
main_command = command.mux_map(
"commands to deploy systems onto the kubernetes cluster", {
"flannel":
command.wrap("deploy the specifications to run flannel",
launch_flannel),
"flannel-monitor":
command.wrap("deploy the specifications to run the flannel monitor",
launch_flannel_monitor),
"dns-addon":
command.wrap("deploy the specifications to run the dns-addon",
launch_dns_addon),
"dns-monitor":
command.wrap("deploy the specifications to run the dns monitor",
launch_dns_monitor),
})
