def get_join_token(self, control_plane_node=False, certificate_key=''): join_token = '' if self.control_plane.ha_masters: token_list = self.instance.self_node_sudo.execute("kubeadm token list").stdout.split('\n') for line in token_list: if 'authentication,signing' in line: print(crayons.white(line)) join_token = line.split()[0].strip() join_url = f'{self.control_plane.apiserver_ip}:{self.control_plane.apiserver_port}' else: join_url = f'{self.instance.allowed_ip}:{self.control_plane.apiserver_port}' join_token = self.instance.self_node_sudo.execute("kubeadm token list | awk '{print $1}'").stdout.split('TOKEN')[-1].strip() while not join_token: logging.warning(crayons.yellow('Join Token not found on master. Creating new join token...')) self.instance.self_node_sudo.execute("kubeadm token create") join_token = self.instance.self_node_sudo.execute("kubeadm token list | awk '{print $1}'").stdout.split('TOKEN')[-1].strip() cert_hash = self.instance.self_node.execute("openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'").stdout.strip() if not join_token or not cert_hash: logging.error(crayons.red('Unable to retrieve join-token or cert-hash')) return None return ( f'kubeadm join {join_url} --token {join_token} --discovery-token-ca-cert-hash sha256:{cert_hash} --control-plane --certificate-key {certificate_key}' ) if control_plane_node else ( f'kubeadm join {join_url} --token {join_token} --discovery-token-ca-cert-hash sha256:{cert_hash}' )
def write_config_to_file(file, local_file_path, script_body): try: with open(local_file_path, mode='w') as local_script_fp: local_script_fp.write('\n'.join(script_body)) return file, local_file_path, None except (OSError, IOError) as file_write_error: logging.error(crayons.red(file_write_error)) return None, None, file_write_error
def supported_cnis(networking='weave'): if networking not in CNI.keys(): logging.error(crayons.red(f'CNI option: {networking} not supported')) return None if networking == 'calico': return CNIDefinitions( cni_url=CNI.get(networking), pod_network_cidr = '192.168.0.0/16', networking_option = f'--pod-network-cidr=192.168.0.0/16', file = 'calico.yaml' ) return CNIDefinitions(cni_url=CNI.get(networking))
def bootstrap_control_plane(self, version='1.16'): cni_definitions = self.supported_cnis(networking=self.control_plane.networking) print(crayons.cyan(f'Building K8s Control-Plane using High Availability: {self.control_plane.ha_masters}')) if self.control_plane.ha_masters: if not self.control_plane.apiserver_ip: logging.warning(crayons.yellow(f'API Server IP must be provided for HA Control Plane option')) return None print(crayons.cyan(f'Getting Container Networking Definitions for CNI: {self.control_plane.networking}')) self.instance.self_node_sudo.execute(f'mkdir -p {self.remote_path}') self.instance.self_node_sudo.execute(f'chown -R $USER:$USER {self.remote_path}') if self.control_plane.networking == 'calico': self.instance.self_node.execute(f'wget {cni_definitions.cni_url} -O {os.path.join(self.remote_path, cni_definitions.file)}') print(crayons.cyan('Pulling Required Images from gcr.io')) has_patch, _, _, _ = semver_has_patch_suffix(version) image_version = version if has_patch else f'stable-{version}' version_snippet = f"--kubernetes-version {image_version.split('-')[0]}" if '-' in image_version else f'--kubernetes-version {image_version}' try: self.instance.self_node_sudo.execute(f'kubeadm config images pull {version_snippet}', warn=True) except invoke.exceptions.UnexpectedExit: logging.warning(crayons.yellow(f'Version: {version_snippet} does not exist.')) version_snippet = '' self.instance.self_node_sudo.execute('kubeadm config images pull') print(crayons.blue('Running pre-flight checks & deploying Control Plane')) init_command = ( f'kubeadm init --control-plane-endpoint "{self.control_plane.apiserver_ip}:{self.control_plane.apiserver_port}" --upload-certs {cni_definitions.networking_option} {version_snippet}' ) if self.control_plane.ha_masters else ( f'kubeadm init {cni_definitions.networking_option} {version_snippet}' ) deployed = self.instance.self_node_sudo.execute(init_command) if deployed.failed: logging.error(crayons.red(f'Master {self.instance.vm_attributes.name} initialization was not performed correctly.')) self.rollback_node() return None print(crayons.green('Initial master deployment success.')) time.sleep(60) self.post_install_steps() if not self.deploy_container_networking(cni_definitions): logging.error(crayons.red(f'Container networking {self.control_plane.networking} failed to deploy correctly.')) return None master_executor = KubeExecutor(wrapper=self.instance.self_node) master_executor.wait_for_running_system_status(namespace='kube-system', remote=True) if self.control_plane.ha_masters: return self.get_certificate_key(deployed) return None
def install_control_plane_loadbalancer(self, is_leader=False): if not self.control_plane.ha_masters: logging.warning(crayons.yellow('Skip install keepalived. Control Plane not Deployed in High Available Mode.')) return None local = LOCAL host = self.instance.vm_attributes.name remote_path = '/etc/keepalived' state = 'MASTER' if is_leader else 'BACKUP' priority = '101' if is_leader else '100' interface = self.get_instance_interface() virtual_ip = self.get_control_plane_virtual_ip() script_file, local_script_path, script_error = self.generate_keepalived_healthcheck(virtual_ip) config_file, local_config_path, config_error = self.generate_keepalived_config( virtual_ip=virtual_ip, interface=interface, state=state, priority=priority ) if script_error or config_error: logging.error(crayons.red(f'Abort keepalived install on {host}')) return None print(crayons.cyan(f'Sending config files to {host}')) sent1 = local.run(f'scp {local_config_path} {host}:~') sent2 = local.run(f'scp {local_script_path} {host}:~') if sent1.ok and sent2.ok: self.wait_pkg_lock() print(crayons.blue(f'Installing keepalived service on {host}')) self.check_install_prerequisites() self.instance.self_node.execute(f'sudo mv {config_file} {remote_path} && sudo mv {script_file} {remote_path}') self.instance.self_node_sudo.execute(f'chmod 0666 {remote_path}/{config_file}') restart = self.instance.self_node_sudo.execute(f'systemctl restart keepalived') if restart.ok: self.instance.self_node_sudo.execute('systemctl status keepalived') test_connection = f'if nc -v -w 5 {virtual_ip} {self.control_plane.apiserver_port}; then echo "Success"; fi' output = self.instance.self_node.execute(test_connection).stderr if 'Connection refused' in output.strip(): print(crayons.green(f'Keepalived running on {host} with virtual ip: {virtual_ip}')) return virtual_ip
def helm_install_v2(self, patch=True, helm=True, tiller=True): prepend = f'HOME={self.home}' helm_script = 'https://git.io/get_helm.sh' current_context = self.get_current_context() if not current_context: return if helm: print(crayons.cyan('Installing Helm locally')) install = self.local.run(f'{prepend} curl -L {helm_script} | bash') if not install.ok: logging.error(crayons.red(f'Helm installation failed')) return self.local.run(f'echo "source <(helm completion bash)" >> {self.home}/.bashrc') print(crayons.green('Helm installed locally')) if tiller: if not patch: logging.warning(crayons.yellow('No-Patch (K8s versions > 1.16.*) installation is not implemented.')) return print(crayons.cyan('Bootstrapping Tiller with patch for K8s versions > 1.16.*')) bootstrap = self.tiller_install_v2_patch() if not bootstrap.ok: logging.error(crayons.red(f'Helm initialization with Tiller failed')) logging.warning(crayons.yellow('Rolling back installation')) rollback = self.local.run(f'{prepend} helm reset --force --remove-helm-home') if rollback.ok: print(crayons.green('Rollback completed')) return tiller_ready = '' while not tiller_ready: print(crayons.white('Ping for tiller ready')) tiller_ready = self.local.run(f'{prepend} kubectl get pod --namespace kube-system -l app=helm,name=tiller --field-selector=status.phase=Running').stdout.strip() time.sleep(1) print(crayons.green(f'Helm initialized with Tiller for context: {current_context}')) self.wait_for_running_system_status() time.sleep(10) print(crayons.magenta('You might need to run "helm init --client-only to initialize repos"')) self.local.run(f'{prepend} helm init --client-only')
def rollback_node(self): logging.warning(crayons.yellow(f'Performing Node {self.instance.vm_attributes.name} Rollback.')) rollback = self.instance.self_node_sudo.execute('kubeadm reset -f --v=5') config_reset = self.instance.self_node.execute(f'rm -f $HOME/.kube/config', warn=True) iptables_reset = self.instance.self_node_sudo.execute('su - root -c \'iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X\'') if rollback.ok: print(crayons.green('Rollback completed.')) else: logging.error(crayons.red('Rollback failed')) return if config_reset.ok: print(crayons.green('Config removed.')) else: logging.warning(crayons.yellow('Config removal not performed.')) if iptables_reset.ok: print(crayons.green('IPTables reset completed.')) else: logging.error(crayons.red('IPTables reset failed.')) return
def deploy_dashboard(self, remote_path='/opt/kube', local=True): """ Argument remote_path exists on hosts with the file by previously running install_kube() method. """ prepend = f'HOME={self.home} ' if local else '' run = self.local.run if local else self.wrapper.execute bootstrap_path = os.path.join(BASE_PATH, 'bootstrap') if local else os.path.join(remote_path, 'bootstrap') user_creation = f'{bootstrap_path}/dashboard-adminuser.yaml' print(crayons.cyan('Deploying Dashboard')) dashboard = run(command=f'{prepend}kubectl apply -f {KUBE_DASHBOARD_URL}') if dashboard.ok: print(crayons.green(f'Kubernetes Dashboard deployed successfully.')) print(crayons.cyan('Deploying User admin-user with role-binding: cluster-admin')) time.sleep(30) user_role = run(command=f'{prepend}kubectl apply -f {user_creation}') if user_role.ok: print(crayons.green(f'User admin-user created successfully.')) else: logging.error(crayons.red('User admin-user was not created correctly.')) else: logging.error(crayons.red('Dashboard was not deployed correctly.'))
def metallb_install(self, file='', version='0.12.0', interface='vmbr0'): prepend = f'HOME={self.home}' if not file: values_file = 'values.yaml' local_workdir = os.path.join(WORKDIR, '.metallb') local_values_path = os.path.join(local_workdir, values_file) metallb_range = pve_cluster_config_client.loadbalancer_ip_range_to_string_or_list() if not metallb_range: logging.error(crayons.red('Could not deploy MetalLB with given cluster values.')) return common_iface = self.get_bridge_common_interface(interface) if not common_iface: logging.error(crayons.red(f'Interface {interface} not found as common bridge in PVE Cluster nodes.')) return template_values = ( 'configInline:', ' address-pools:', f' - name: {common_iface}', ' protocol: layer2', ' addresses:', f' - {metallb_range}', 'controller:', ' tolerations:', ' - effect: NoExecute', ' key: node.kubernetes.io/not-ready', ' operator: Exists', ' tolerationSeconds: 60', ' - effect: NoExecute', ' key: node.kubernetes.io/unreachable', ' operator: Exists', ' tolerationSeconds: 60', ) self.local.run(f'mkdir -p {local_workdir}') _, file, error = KubeProvisioner.write_config_to_file( file=values_file, local_file_path=local_values_path, script_body=template_values ) if error: # Logging executes in write_config_to_file return if not self.helm_exists(prepend): return print(crayons.cyan(f'Deploying MetalLB')) metal_install = self.local.run(f'{prepend} helm install --name metallb -f {file} stable/metallb --version={version}') if metal_install.failed: logging.error(crayons.red('MetalLB installation failed.Rolling Back')) rollback = self.local.run('helm delete metallb --purge') if rollback.ok: print(crayons.green('Rollback completed')) return print(crayons.green('MetalLB installed'))
def join_node(self, leader: InstanceClone, control_plane_node=False, certificate_key=''): leader_provisioner = KubeProvisioner.kube_provisioner_factory(os_type=leader.vm_attributes.os_type)( instance=leader, control_plane=self.control_plane, remote_path=self.remote_path ) if not certificate_key or not leader.allowed_ip: join_command = leader_provisioner.get_join_token_v2(control_plane_node) else: join_command = leader_provisioner.get_join_token(control_plane_node, certificate_key) if not join_command: logging.error(crayons.red('Node Join command not generated. Abort.')) return print(crayons.white(f'Join command: {join_command}')) print(crayons.cyan(f'Joining Node: {self.instance.vm_attributes.name} to the cluster')) join = self.instance.self_node_sudo.execute(join_command) if join.failed: logging.error(crayons.red(f'Joining Node: {self.instance.vm_attributes.name} failed. Performing Rollback.')) self.rollback_node() return if control_plane_node: self.post_install_steps() print(crayons.green(f'Node: {self.instance.vm_attributes.name} has joined the cluster.'))
def add_local_cluster_config( self, custom_cluster_name=None, custom_context=None, custom_user_name=None, set_current_context=True ): # TODO: Needs further refactoring. local_kube = os.path.join(self.home, '.kube') remote_kube = os.path.join(self.remote, '.kube') local_config_base = os.path.join(local_kube, 'config') remote_config_base = os.path.join(remote_kube, 'config') local_dump_folder = os.path.join(WORKDIR, 'dump') local_dump = os.path.join(local_dump_folder, self.host, 'config.yaml') self.local.run(f'mkdir -p {local_dump_folder} && mkdir -p {os.path.join(local_dump_folder, self.host)}') self.local.run(f'scp {self.host}:{remote_config_base} {local_dump}') remote_kube_config = KubeConfig( config_filename=local_dump, custom_cluster_name=custom_cluster_name, custom_context=custom_context, custom_user_name=custom_user_name, set_current_context=set_current_context ) remote_config_yaml = remote_kube_config.serialize_config() self.local.run(f'rm -rf {local_dump_folder}') current_context = remote_kube_config.current_context print(crayons.cyan('Current Context: ') + f'{current_context}') remote_cluster = remote_kube_config.cluster_first remote_user = remote_kube_config.user_first remote_context = remote_kube_config.context_first remote_cluster_name, _, _ = KubeConfig.get_cluster_data(remote_cluster) remote_user_name, _, _ = KubeConfig.get_user_data(remote_user) self.local.run(f'cp {local_config_base} {local_config_base}.bak') local_kube_config = KubeConfig( config_filename=local_config_base, custom_cluster_name=custom_cluster_name, custom_context=custom_context, custom_user_name=custom_user_name, set_current_context=set_current_context ) local_config_yaml = local_kube_config.serialize_config() local_kube_config.update_current_context(current_context) custom_cluster_name_used = True if custom_cluster_name else False custom_user_name_used = True if custom_user_name else False local_kube_config.get_or_create_custom_cluster_user_values( new_cluster_name=remote_cluster_name, new_user_name=remote_user_name ) local_cluster_entries = local_kube_config.clusters local_context_entries = local_kube_config.contexts local_user_entries = local_kube_config.users for cluster_entry in local_cluster_entries: if cluster_entry.get('name') == remote_cluster_name or custom_cluster_name_used: remote_cluster['name'] = custom_cluster_name for user_entry in local_user_entries: if user_entry.get('name') == remote_user_name or custom_user_name_used: remote_user['name'] = custom_user_name for context_entry in local_context_entries: if context_entry.get('context').get('cluster') == remote_cluster_name or custom_cluster_name_used: remote_context['context']['cluster'] = custom_cluster_name if context_entry.get('context').get('user') == remote_user_name or custom_user_name_used: remote_context['context']['user'] = custom_user_name remote_context['name'] = ( custom_context if custom_context else f'{remote_user["name"]}@{remote_cluster["name"]}' ) if not local_kube_config.clusters: local_config_yaml['clusters'] = remote_config_yaml.get('clusters') else: local_config_yaml['clusters'].append(remote_cluster) if not local_kube_config.users: local_config_yaml['users'] = remote_config_yaml.get('users') else: local_config_yaml['users'].append(remote_user) if not local_kube_config.contexts: local_config_yaml['contexts'] = remote_config_yaml.get('contexts') else: local_config_yaml['contexts'].append(remote_context) local_config_yaml['current-context'] = ( custom_context if set_current_context and custom_context else local_kube_config.current_context ) try: with open(local_config_base, mode='w') as local_config_file_mutated: try: yaml.safe_dump(local_config_yaml, stream=local_config_file_mutated) except yaml.YAMLError as yaml_error: logging.error(crayons.red(f'Error: failed to load from {local_config_base}')) logging.error(crayons.red(f'{yaml_error}')) print(crayons.blue(f'Performing rollback of {local_config_base}')) self.local.run(f'mv {local_config_base}.bak {local_config_base}') print(crayons.green('Rollback complete')) return except Exception as generic: logging.error(crayons.red(f'Error during writing to kube config {local_config_base}: {generic}')) print(crayons.blue(f'Performing rollback of {local_config_base}')) self.local.run(f'mv {local_config_base}.bak {local_config_base}') print(crayons.green('Rollback complete'))