def auto_node(ops: setup.Operations, tc: TerminationContext,
node: configuration.Node, install_iso: str):
vm = VirtualMachine(node, tc, install_iso)
ops.add_operation("install node @HOST (this may take several minutes)",
vm.boot_install_and_admit, node)
vm = VirtualMachine(node, tc)
ops.add_operation("start up node @HOST", lambda: vm.boot_launch(), node)
def sequence_keysystem(ops: setup.Operations) -> None:
ops.add_subcommand(setup.setup_keyserver)
ops.add_operation("verify that keyserver static files can be fetched",
iterative_verifier(verify.check_keystatics, 10.0))
ops.add_subcommand(setup.admit_keyserver)
ops.add_subcommand(setup.setup_keygateway)
ops.add_operation("verify that the keygateway is responsive",
verify.check_keygateway)
def ssh_foreach(ops: setup.Operations, node_kind: str, *params: str):
config = configuration.get_config()
valid_node_kinds = configuration.Node.VALID_NODE_KINDS
if not (node_kind == "node" or node_kind in valid_node_kinds):
command.fail("usage: spire foreach {node," +
",".join(valid_node_kinds) + "} command")
for node in config.nodes:
if node_kind == "node" or node.kind == node_kind:
ops.ssh("run command on @HOST", node, *params)
def sequence_core(ops: setup.Operations, config: configuration.Config) -> None:
# ** Install and upgrade packages on all systems **
# spire infra install-packages
infra.infra_install_packages(ops, config)
# ** Launch services **
# spire setup services
setup.setup_services(ops, config)
# spire verify etcd
ops.add_operation("verify that etcd has launched successfully",
iterative_verifier(verify.check_etcd_health, 20.0))
# spire verify kubernetes
ops.add_operation("verify that kubernetes has launched successfully",
iterative_verifier(verify.check_kube_health, 10.0))
def sequence_ssh(ops: setup.Operations, config: configuration.Config) -> None:
# ** Request SSH cert **
# spire access ssh
# (if this fails, you might need to make sure you don't have any stale kerberos tickets)
ops.add_operation("request SSH access to cluster",
access.access_ssh_with_add)
# ** Configure and test SSH **
# spire setup supervisor-ssh
setup.setup_supervisor_ssh(ops, config)
# spire verify ssh-with-certs
ops.add_operation("verify ssh access to supervisor",
verify.check_ssh_with_certs)
def sequence_core(ops: setup.Operations) -> None:
ops.add_subcommand(setup.setup_services)
ops.add_operation("verify that etcd has launched successfully",
iterative_verifier(verify.check_etcd_health, 20.0))
ops.add_operation("verify that kubernetes has launched successfully",
iterative_verifier(verify.check_kube_health, 10.0))
ops.print_annotations("set up the core kubernetes cluster")
def sequence_keysystem(ops: setup.Operations,
config: configuration.Config) -> None:
# ** Configure the supervisor keyserver **
# spire setup keyserver
setup.setup_keyserver(ops, config)
# spire verify keystatics
ops.add_operation("verify that keyserver static files can be fetched",
verify.check_keystatics)
# ** Admit the supervisor node to the cluster **
# spire setup self-admit
setup.admit_keyserver(ops, config)
# ** Prepare kerberos gateway **
# spire setup keygateway
setup.setup_keygateway(ops, config)
# spire verify keygateway
ops.add_operation("verify that the keygateway is responsive",
verify.check_keygateway)
def sequence_dns_addon(ops: setup.Operations,
config: configuration.Config) -> None:
ops.add_operation("deploy or update dns-addon",
lambda: deploy.launch_spec("dns-addon.yaml"))
ops.add_operation("verify that dns-addon is online",
iterative_verifier(verify.check_dns_kubeinfo, 60.0))
ops.add_operation("verify that dns-addon is functioning",
verify.check_dns_function)
def sequence_flannel(ops: setup.Operations,
config: configuration.Config) -> None:
ops.add_operation("deploy or update flannel",
lambda: deploy.launch_spec("flannel.yaml"))
ops.add_operation("verify that flannel is online",
iterative_verifier(verify.check_flannel_kubeinfo, 60.0))
ops.add_operation("verify that flannel is functioning",
verify.check_flannel_function)
def infra_install_packages(ops: setup.Operations, config: configuration.Config) -> None:
for node in config.nodes:
ops.ssh("update apt repositories on @HOST", node, "apt-get", "update")
ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y")
if node.kind == "supervisor":
ops.ssh("install supervisor packages on @HOST", node, "apt-get", "install", "-y",
"homeworld-bootstrap-registry")
else:
ops.ssh("install standard packages on @HOST", node, "apt-get", "install", "-y",
"homeworld-services")
def auto_supervisor(ops: setup.Operations, tc: TerminationContext,
supervisor: configuration.Node, install_iso: str):
vm = VirtualMachine(supervisor, tc, install_iso)
ops.add_operation(
"install supervisor node (this may take several minutes)",
vm.boot_install_supervisor, supervisor)
# TODO: annotations, so that this can be --dry-run'd
vm = VirtualMachine(supervisor, tc)
ops.add_operation("start up supervisor node",
lambda: vm.boot_launch(autoadd_fingerprint=True))
ops.add_subcommand(seq.sequence_supervisor)
def sequence_addons(ops: setup.Operations) -> None:
ops.add_operation("deploy or update flannel", deploy.launch_flannel)
ops.add_operation("deploy or update dns-addon", deploy.launch_dns_addon)
ops.add_operation("verify that flannel is online", iterative_verifier(verify.check_flannel_kubeinfo, 60.0))
ops.add_operation("verify that dns-addon is online", iterative_verifier(verify.check_dns_kubeinfo, 60.0))
ops.add_operation("verify that flannel is functioning", verify.check_flannel_function)
ops.add_operation("verify that dns-addon is functioning", verify.check_dns_function)
ops.print_annotations("set up the dns-addon")
def sequence_registry(ops: setup.Operations) -> None:
ops.add_subcommand(setup.setup_dns_bootstrap)
ops.add_subcommand(setup.setup_bootstrap_registry)
ops.add_operation("verify that acis can be pulled from the registry", verify.check_aci_pull)
ops.print_annotations("set up the bootstrap container registry")
def sequence_registry(ops: setup.Operations,
config: configuration.Config) -> None:
setup.setup_dns_bootstrap(ops, config)
setup.setup_bootstrap_registry(ops, config)
ops.add_operation("verify that acis can be pulled from the registry",
verify.check_aci_pull)
def sequence_supervisor(ops: setup.Operations) -> None:
ops.add_subcommand(sequence_keysystem)
ops.add_subcommand(sequence_ssh)
ops.print_annotations("set up the keysystem")
def sequence_ssh(ops: setup.Operations) -> None:
ops.add_operation("request SSH access to cluster", access.access_ssh_with_add)
ops.add_subcommand(setup.setup_supervisor_ssh)
ops.add_operation("verify ssh access to supervisor", verify.check_ssh_with_certs)
ops.print_annotations("set up ssh")
def sequence_cluster(ops: setup.Operations) -> None:
ops.add_operation(
"verify that the fundamental cluster infrastructure is online",
iterative_verifier(verify.check_online, 120.0))
ops.add_operation("verify that etcd has launched successfully",
iterative_verifier(verify.check_etcd_health, 120.0))
ops.add_operation("verify that kubernetes has launched successfully",
iterative_verifier(verify.check_kube_health, 120.0))
ops.add_operation("verify that acis can be pulled from the registry",
iterative_verifier(verify.check_aci_pull, 60.0))
ops.add_operation("verify that flannel is online",
iterative_verifier(verify.check_flannel, 180.0))
ops.add_operation("verify that dns-addon is online",
iterative_verifier(verify.check_dns, 120.0))
def auto_cluster(ops: setup.Operations, authorized_key=None):
if authorized_key is None:
if "HOME" not in os.environ:
command.fail(
"expected $HOME to be set for authorized_key autodetect")
authorized_key = os.path.join(os.getenv("HOME"), ".ssh/id_rsa.pub")
project, config = configuration.get_project(), configuration.get_config()
iso_path = os.path.join(project, "cluster-%d.iso" % os.getpid())
ops.add_operation("check nested virtualization", qemu_check_nested_virt)
ops.add_operation("update known hosts", access.update_known_hosts)
ops.add_operation("generate ISO",
lambda: iso.gen_iso(iso_path, authorized_key, "serial"))
with ops.context("networking", net_context()):
with ops.context("termination", TerminationContext()) as tc:
with ops.context("debug shell", DebugContext()):
ops.add_subcommand(lambda ops: auto_supervisor(
ops, tc, config.keyserver, iso_path))
for node in config.nodes:
if node == config.keyserver: continue
ops.add_subcommand(
lambda ops, n=node: auto_node(ops, tc, n, iso_path))
ops.add_subcommand(seq.sequence_cluster)
def sequence_ssh(ops: setup.Operations) -> None:
ops.add_operation("request SSH access to cluster",
access.access_ssh_with_add)
ops.add_subcommand(setup.setup_supervisor_ssh)
ops.add_operation("verify ssh access to supervisor",
iterative_verifier(verify.check_ssh_with_certs, 20.0))
def sequence_supervisor(ops: setup.Operations) -> None:
ops.add_subcommand(sequence_keysystem)
ops.add_operation(
"verify that keysystem certs are available on supervisor",
iterative_verifier(verify.check_certs_on_supervisor, 20.0))
ops.add_subcommand(setup.setup_prometheus)
ops.add_subcommand(sequence_ssh)
ops.add_subcommand(setup.setup_bootstrap_registry)
ops.add_operation("pre-deploy flannel", deploy.launch_flannel)
ops.add_operation("pre-deploy dns-addon", deploy.launch_dns_addon)
ops.add_operation("pre-deploy flannel-monitor",
deploy.launch_flannel_monitor)
ops.add_operation("pre-deploy dns-monitor", deploy.launch_dns_monitor)
ops.add_operation("pre-deploy user-grant", deploy.launch_user_grant)
# TODO: have a way to do this without a specialized just-for-supervisor method
ops.add_subcommand(infra.infra_sync_supervisor)
def sequence_keysystem(ops: setup.Operations) -> None:
ops.add_subcommand(setup.setup_keyserver)
ops.add_operation("verify that keyserver static files can be fetched",
iterative_verifier(verify.check_keystatics, 10.0))
ops.add_subcommand(setup.admit_keyserver)
if configuration.get_config().is_kerberos_enabled():
ops.add_subcommand(setup.setup_keygateway)
ops.add_operation("verify that the keygateway is responsive",
verify.check_keygateway)
else:
ops.add_operation("skip keygateway enablement (kerberos is disabled)",
lambda: None)
def infra_sync(ops: setup.Operations, node_name: str) -> None:
node = configuration.get_config().get_node(node_name)
ops.ssh("synchronize operations on @HOST", node, "sync")
def infra_install_packages(ops: setup.Operations) -> None:
config = configuration.get_config()
for node in config.nodes:
ops.ssh("update apt repositories on @HOST", node, "apt-get", "update")
ops.ssh("upgrade packages on @HOST", node, "apt-get", "upgrade", "-y")
def sequence_supervisor(ops: setup.Operations) -> None:
ops.add_subcommand(sequence_keysystem)
ops.add_operation(
"verify that kubernetes certs are available on supervisor",
iterative_verifier(verify.check_certs_on_supervisor, 20.0))
ops.add_subcommand(setup.setup_prometheus)
ops.add_subcommand(sequence_ssh)
ops.add_subcommand(setup.setup_bootstrap_registry)
ops.add_operation("pre-deploy flannel", deploy.launch_flannel)
ops.add_operation("pre-deploy dns-addon", deploy.launch_dns_addon)
ops.add_operation("pre-deploy flannel-monitor",
deploy.launch_flannel_monitor)
ops.add_operation("pre-deploy dns-monitor", deploy.launch_dns_monitor)