def auto_install_nodes(ops: command.Operations, tc: TerminationContext, nodes: list, install_iso: str, cdrom_install: bool = False, debug_qemu=False): vms = [ VirtualMachine(node, tc, install_iso, cdrom_install=cdrom_install, debug_qemu=debug_qemu) for node in nodes ] def boot_install_and_admit_all(): with concurrent.futures.ThreadPoolExecutor(len(vms)) as executor: futures = [ executor.submit(vm.boot_install_and_admit) for vm in vms ] for f in futures: f.result() ops.add_operation( "install non-supervisor nodes (this may take several minutes)", boot_install_and_admit_all)
def modify_keygateway(ops: command.Operations, overwrite_keytab: bool) -> None: config = configuration.get_config() if not config.is_kerberos_enabled(): print("keygateway disabled; skipping") return for node in config.nodes: if node.kind != "supervisor": continue # keytab is stored encrypted in the configuration folder keytab = os.path.join(configuration.get_project(), "keytab.%s.crypt" % node.hostname) decrypted = keycrypt.gpg_decrypt_to_memory(keytab) def safe_upload_keytab(node=node): if not overwrite_keytab: try: existing_keytab = ssh.check_ssh_output(node, "cat", KEYTAB_PATH) except subprocess.CalledProcessError as e_test: # if there is no existing keytab, cat will fail with error code 1 if e_test.returncode != 1: command.fail(e_test) print("no existing keytab found, uploading local keytab") else: if existing_keytab != decrypted: command.fail("existing keytab does not match local keytab") return # existing keytab matches local keytab, no action required ssh.upload_bytes(node, decrypted, KEYTAB_PATH) ops.add_operation("upload keytab for {}".format(node), safe_upload_keytab) ssh_cmd(ops, "enable keygateway on @HOST", node, "systemctl", "enable", "keygateway") ssh_cmd(ops, "restart keygateway on @HOST", node, "systemctl", "restart", "keygateway")
def auto_launch_nodes(ops: command.Operations, tc: TerminationContext, nodes: list, debug_qemu=False): for node in nodes: vm = VirtualMachine(node, tc, debug_qemu=debug_qemu) ops.add_operation("start up node {}".format(node), vm.boot_launch)
def sequence_supervisor(ops: command.Operations, skip_verify_keygateway: bool = False) -> None: "set up and verify functionality of entire supervisor node (keysystem + ssh)" config = configuration.get_config() ops.add_subcommand(sequence_keysystem, skip_verify_keygateway=skip_verify_keygateway) ops.add_command(iterative_verifier(verify.check_certs_on_supervisor, 20.0)) ops.add_subcommand(setup.setup_prometheus) ops.add_subcommand(sequence_ssh) ops.add_subcommand(setup.setup_bootstrap_registry) ops.add_subcommand(setup.update_registry) ops.add_command(deploy.launch_flannel) ops.add_command(deploy.launch_dns_addon) ops.add_command(deploy.launch_flannel_monitor) ops.add_command(deploy.launch_dns_monitor) if config.user_grant_domain != '': ops.add_command(deploy.launch_user_grant) else: ops.add_operation("skip pre-deploying user-grant (not configured)", lambda: None) for node in config.nodes: if node.kind == 'supervisor': ops.add_subcommand(infra.infra_sync, node.hostname)
def auto_launch_supervisor(ops: command.Operations, tc: TerminationContext, supervisor: configuration.Node, debug_qemu=False): # TODO: annotations, so that this can be --dry-run'd vm = VirtualMachine(supervisor, tc, debug_qemu=debug_qemu) ops.add_operation("start up supervisor node", lambda: vm.boot_launch(autoadd_fingerprint=True))
def auto_install_supervisor(ops: command.Operations, tc: TerminationContext, supervisor: configuration.Node, install_iso: str, cdrom_install: bool = False, debug_qemu=False): vm = VirtualMachine(supervisor, tc, install_iso, cdrom_install=cdrom_install, debug_qemu=debug_qemu) ops.add_operation( "install supervisor node (this may take several minutes)", vm.boot_install_supervisor, supervisor)
def sequence_keysystem(ops: command.Operations, skip_verify_keygateway: bool = False) -> None: "set up and verify functionality of the keyserver and keygateway" ops.add_command( iterative_verifier(verify.check_supervisor_accessible, 30.0)) ops.add_subcommand(setup.setup_keyserver) ops.add_command(iterative_verifier(verify.check_keystatics, 60.0)) ops.add_subcommand(setup.admit_keyserver) if configuration.get_config().is_kerberos_enabled(): ops.add_subcommand(setup.setup_keygateway) if not skip_verify_keygateway: ops.add_command(verify.check_keygateway) else: ops.add_operation("skip keygateway verification", lambda: None) else: ops.add_operation("skip keygateway enablement (kerberos is disabled)", lambda: None)
def sequence_cluster(ops: command.Operations) -> None: "set up and verify kubernetes infrastructure operation" ops.add_command(iterative_verifier(verify.check_online, 120.0)) ops.add_command(iterative_verifier(verify.check_systemd_services, 120.0)) ops.add_command(iterative_verifier(verify.check_etcd_health, 120.0)) ops.add_command(iterative_verifier(verify.check_kube_health, 120.0)) ops.add_command(iterative_verifier(verify.check_pull, 120.0)) ops.add_command(iterative_verifier(verify.check_flannel_pods, 210.0)) ops.add_command(iterative_verifier(verify.check_exec, 120.0)) ops.add_command(iterative_verifier(verify.check_flannel, 120.0)) ops.add_command(iterative_verifier(verify.check_dns, 120.0)) if configuration.get_config().user_grant_domain == '': ops.add_operation("skip verifying user-grant (not configured)", lambda: None) elif not verify.is_user_grant_verifiable(): ops.add_operation("skip verifying user-grant (no client certificate)", lambda: None) else: ops.add_operation("verify that user-grant is working properly", iterative_verifier(verify.check_user_grant, 120.0))
def auto_install(ops: command.Operations, authorized_key=None, persistent: bool = False, cdrom_install: bool = False, debug_qemu: bool = False): "complete cluster installation and launch" if authorized_key is None: if "HOME" not in os.environ: command.fail( "expected $HOME to be set for authorized_key autodetect") authorized_key = os.path.join(os.getenv("HOME"), ".ssh/id_rsa.pub") project, config = configuration.get_project(), configuration.get_config() iso_path = os.path.join(project, "cluster-%d.iso" % os.getpid()) ops.add_operation("check nested virtualization", qemu_check_nested_virt) ops.add_operation("update known hosts", access.update_known_hosts) ops.add_operation("generate ISO", lambda: iso.gen_iso(iso_path, authorized_key, "serial")) with ops.context("networking", net_context()): with ops.context("termination", TerminationContext()) as tc: with ops.context("debug shell", DebugContext(persistent)): ops.add_subcommand(auto_install_supervisor, tc, config.keyserver, iso_path, cdrom_install=cdrom_install, debug_qemu=debug_qemu) ops.add_subcommand(auto_launch_supervisor, tc, config.keyserver, debug_qemu=debug_qemu) ops.add_subcommand(seq.sequence_supervisor) other_nodes = [ n for n in config.nodes if n != config.keyserver ] ops.add_subcommand(auto_install_nodes, tc, other_nodes, iso_path, cdrom_install=cdrom_install, debug_qemu=debug_qemu) ops.add_subcommand(auto_launch_nodes, tc, other_nodes, debug_qemu=debug_qemu) ops.add_subcommand(seq.sequence_cluster)