def auto_launch_nodes(ops: command.Operations,
tc: TerminationContext,
nodes: list,
debug_qemu=False):
for node in nodes:
vm = VirtualMachine(node, tc, debug_qemu=debug_qemu)
ops.add_operation("start up node {}".format(node), vm.boot_launch)
def modify_keygateway(ops: command.Operations, overwrite_keytab: bool) -> None:
config = configuration.get_config()
if not config.is_kerberos_enabled():
print("keygateway disabled; skipping")
return
for node in config.nodes:
if node.kind != "supervisor":
continue
# keytab is stored encrypted in the configuration folder
keytab = os.path.join(configuration.get_project(), "keytab.%s.crypt" % node.hostname)
decrypted = keycrypt.gpg_decrypt_to_memory(keytab)
def safe_upload_keytab(node=node):
if not overwrite_keytab:
try:
existing_keytab = ssh.check_ssh_output(node, "cat", KEYTAB_PATH)
except subprocess.CalledProcessError as e_test:
# if there is no existing keytab, cat will fail with error code 1
if e_test.returncode != 1:
command.fail(e_test)
print("no existing keytab found, uploading local keytab")
else:
if existing_keytab != decrypted:
command.fail("existing keytab does not match local keytab")
return # existing keytab matches local keytab, no action required
ssh.upload_bytes(node, decrypted, KEYTAB_PATH)
ops.add_operation("upload keytab for {}".format(node), safe_upload_keytab)
ssh_cmd(ops, "enable keygateway on @HOST", node, "systemctl", "enable", "keygateway")
ssh_cmd(ops, "restart keygateway on @HOST", node, "systemctl", "restart", "keygateway")
def auto_install_nodes(ops: command.Operations,
tc: TerminationContext,
nodes: list,
install_iso: str,
cdrom_install: bool = False,
debug_qemu=False):
vms = [
VirtualMachine(node,
tc,
install_iso,
cdrom_install=cdrom_install,
debug_qemu=debug_qemu) for node in nodes
]
def boot_install_and_admit_all():
with concurrent.futures.ThreadPoolExecutor(len(vms)) as executor:
futures = [
executor.submit(vm.boot_install_and_admit) for vm in vms
]
for f in futures:
f.result()
ops.add_operation(
"install non-supervisor nodes (this may take several minutes)",
boot_install_and_admit_all)
def auto_launch_supervisor(ops: command.Operations,
tc: TerminationContext,
supervisor: configuration.Node,
debug_qemu=False):
# TODO: annotations, so that this can be --dry-run'd
vm = VirtualMachine(supervisor, tc, debug_qemu=debug_qemu)
ops.add_operation("start up supervisor node",
lambda: vm.boot_launch(autoadd_fingerprint=True))
def auto_install_supervisor(ops: command.Operations,
tc: TerminationContext,
supervisor: configuration.Node,
install_iso: str,
cdrom_install: bool = False,
debug_qemu=False):
vm = VirtualMachine(supervisor,
tc,
install_iso,
cdrom_install=cdrom_install,
debug_qemu=debug_qemu)
ops.add_operation(
"install supervisor node (this may take several minutes)",
vm.boot_install_supervisor, supervisor)
def auto_launch(ops: command.Operations, debug_qemu: bool = False):
"launch installed cluster"
config = configuration.get_config()
with ops.context("networking", net_context()):
with ops.context("termination", TerminationContext()) as tc:
with ops.context("debug shell", DebugContext(True)):
ops.add_subcommand(auto_launch_supervisor,
tc,
config.keyserver,
debug_qemu=debug_qemu)
other_nodes = [
n for n in config.nodes if n != config.keyserver
]
ops.add_subcommand(auto_launch_nodes,
tc,
other_nodes,
debug_qemu=debug_qemu)
def auto_install(ops: command.Operations,
authorized_key=None,
persistent: bool = False,
cdrom_install: bool = False,
debug_qemu: bool = False):
"complete cluster installation and launch"
if authorized_key is None:
if "HOME" not in os.environ:
command.fail(
"expected $HOME to be set for authorized_key autodetect")
authorized_key = os.path.join(os.getenv("HOME"), ".ssh/id_rsa.pub")
project, config = configuration.get_project(), configuration.get_config()
iso_path = os.path.join(project, "cluster-%d.iso" % os.getpid())
ops.add_operation("check nested virtualization", qemu_check_nested_virt)
ops.add_operation("update known hosts", access.update_known_hosts)
ops.add_operation("generate ISO",
lambda: iso.gen_iso(iso_path, authorized_key, "serial"))
with ops.context("networking", net_context()):
with ops.context("termination", TerminationContext()) as tc:
with ops.context("debug shell", DebugContext(persistent)):
ops.add_subcommand(auto_install_supervisor,
tc,
config.keyserver,
iso_path,
cdrom_install=cdrom_install,
debug_qemu=debug_qemu)
ops.add_subcommand(auto_launch_supervisor,
tc,
config.keyserver,
debug_qemu=debug_qemu)
ops.add_subcommand(seq.sequence_supervisor)
other_nodes = [
n for n in config.nodes if n != config.keyserver
]
ops.add_subcommand(auto_install_nodes,
tc,
other_nodes,
iso_path,
cdrom_install=cdrom_install,
debug_qemu=debug_qemu)
ops.add_subcommand(auto_launch_nodes,
tc,
other_nodes,
debug_qemu=debug_qemu)
ops.add_subcommand(seq.sequence_cluster)
def sequence_supervisor(ops: command.Operations,
skip_verify_keygateway: bool = False) -> None:
"set up and verify functionality of entire supervisor node (keysystem + ssh)"
config = configuration.get_config()
ops.add_subcommand(sequence_keysystem,
skip_verify_keygateway=skip_verify_keygateway)
ops.add_command(iterative_verifier(verify.check_certs_on_supervisor, 20.0))
ops.add_subcommand(setup.setup_prometheus)
ops.add_subcommand(sequence_ssh)
ops.add_subcommand(setup.setup_bootstrap_registry)
ops.add_subcommand(setup.update_registry)
ops.add_command(deploy.launch_flannel)
ops.add_command(deploy.launch_dns_addon)
ops.add_command(deploy.launch_flannel_monitor)
ops.add_command(deploy.launch_dns_monitor)
if config.user_grant_domain != '':
ops.add_command(deploy.launch_user_grant)
else:
ops.add_operation("skip pre-deploying user-grant (not configured)",
lambda: None)
for node in config.nodes:
if node.kind == 'supervisor':
ops.add_subcommand(infra.infra_sync, node.hostname)
def sequence_ssh(ops: command.Operations) -> None:
"set up and verify ssh access to the supervisor node"
ops.add_command(access.access_ssh)
ops.add_subcommand(setup.setup_supervisor_ssh)
ops.add_command(iterative_verifier(verify.check_ssh_with_certs, 20.0))
def sequence_keysystem(ops: command.Operations,
skip_verify_keygateway: bool = False) -> None:
"set up and verify functionality of the keyserver and keygateway"
ops.add_command(
iterative_verifier(verify.check_supervisor_accessible, 30.0))
ops.add_subcommand(setup.setup_keyserver)
ops.add_command(iterative_verifier(verify.check_keystatics, 60.0))
ops.add_subcommand(setup.admit_keyserver)
if configuration.get_config().is_kerberos_enabled():
ops.add_subcommand(setup.setup_keygateway)
if not skip_verify_keygateway:
ops.add_command(verify.check_keygateway)
else:
ops.add_operation("skip keygateway verification", lambda: None)
else:
ops.add_operation("skip keygateway enablement (kerberos is disabled)",
lambda: None)
def sequence_cluster(ops: command.Operations) -> None:
"set up and verify kubernetes infrastructure operation"
ops.add_command(iterative_verifier(verify.check_online, 120.0))
ops.add_command(iterative_verifier(verify.check_systemd_services, 120.0))
ops.add_command(iterative_verifier(verify.check_etcd_health, 120.0))
ops.add_command(iterative_verifier(verify.check_kube_health, 120.0))
ops.add_command(iterative_verifier(verify.check_pull, 120.0))
ops.add_command(iterative_verifier(verify.check_flannel_pods, 210.0))
ops.add_command(iterative_verifier(verify.check_exec, 120.0))
ops.add_command(iterative_verifier(verify.check_flannel, 120.0))
ops.add_command(iterative_verifier(verify.check_dns, 120.0))
if configuration.get_config().user_grant_domain == '':
ops.add_operation("skip verifying user-grant (not configured)",
lambda: None)
elif not verify.is_user_grant_verifiable():
ops.add_operation("skip verifying user-grant (no client certificate)",
lambda: None)
else:
ops.add_operation("verify that user-grant is working properly",
iterative_verifier(verify.check_user_grant, 120.0))