def is_alive(self):
try:
return self.instance.self_node.execute(command='uptime', warn=True).ok
except Exception as notready:
logging.warning(crayons.yellow(notready))
logging.warning(crayons.yellow(f'Retry connection on {self.instance.vm_attributes.name}'))
self.instance.self_node = FabricWrapper(host=self.instance.vm_attributes.name)
self.instance.self_node_sudo = FabricWrapper(host=self.instance.vm_attributes.name, sudo=True)
return False
def bootstrap_control_plane(self, version='1.16'):
cni_definitions = self.supported_cnis(networking=self.control_plane.networking)
print(crayons.cyan(f'Building K8s Control-Plane using High Availability: {self.control_plane.ha_masters}'))
if self.control_plane.ha_masters:
if not self.control_plane.apiserver_ip:
logging.warning(crayons.yellow(f'API Server IP must be provided for HA Control Plane option'))
return None
print(crayons.cyan(f'Getting Container Networking Definitions for CNI: {self.control_plane.networking}'))
self.instance.self_node_sudo.execute(f'mkdir -p {self.remote_path}')
self.instance.self_node_sudo.execute(f'chown -R $USER:$USER {self.remote_path}')
if self.control_plane.networking == 'calico':
self.instance.self_node.execute(f'wget {cni_definitions.cni_url} -O {os.path.join(self.remote_path, cni_definitions.file)}')
print(crayons.cyan('Pulling Required Images from gcr.io'))
has_patch, _, _, _ = semver_has_patch_suffix(version)
image_version = version if has_patch else f'stable-{version}'
version_snippet = f"--kubernetes-version {image_version.split('-')[0]}" if '-' in image_version else f'--kubernetes-version {image_version}'
try:
self.instance.self_node_sudo.execute(f'kubeadm config images pull {version_snippet}', warn=True)
except invoke.exceptions.UnexpectedExit:
logging.warning(crayons.yellow(f'Version: {version_snippet} does not exist.'))
version_snippet = ''
self.instance.self_node_sudo.execute('kubeadm config images pull')
print(crayons.blue('Running pre-flight checks & deploying Control Plane'))
init_command = (
f'kubeadm init --control-plane-endpoint "{self.control_plane.apiserver_ip}:{self.control_plane.apiserver_port}" --upload-certs {cni_definitions.networking_option} {version_snippet}'
) if self.control_plane.ha_masters else (
f'kubeadm init {cni_definitions.networking_option} {version_snippet}'
)
deployed = self.instance.self_node_sudo.execute(init_command)
if deployed.failed:
logging.error(crayons.red(f'Master {self.instance.vm_attributes.name} initialization was not performed correctly.'))
self.rollback_node()
return None
print(crayons.green('Initial master deployment success.'))
time.sleep(60)
self.post_install_steps()
if not self.deploy_container_networking(cni_definitions):
logging.error(crayons.red(f'Container networking {self.control_plane.networking} failed to deploy correctly.'))
return None
master_executor = KubeExecutor(wrapper=self.instance.self_node)
master_executor.wait_for_running_system_status(namespace='kube-system', remote=True)
if self.control_plane.ha_masters:
return self.get_certificate_key(deployed)
return None
def helm_exists(self, command_prefix):
exit_code = self.local.run(f'{command_prefix} helm version ; echo $?').stdout.split()[-1].strip()
if exit_code != '0':
logging.warning(crayons.yellow('"helm" not found on your system. Please run helm-install first'))
return exit_code == '0'
# TODO: Storage feature.
def remove_cluster_node(self, instance: InstanceClone):
print(crayons.cyan(f'Removing node {instance.vm_attributes.name}'))
remove = self.local.run(f'HOME={self.home} kubectl delete node {instance.vm_attributes.name}', warn=True)
if remove.ok:
print(crayons.green(f'Node {instance.vm_attributes.name} removed from cluster.'))
else:
logging.warning(crayons.yellow(f'Node {instance.vm_attributes.name} was not found or not removed from cluster.'))
def get_join_token(self, control_plane_node=False, certificate_key=''):
join_token = ''
if self.control_plane.ha_masters:
token_list = self.instance.self_node_sudo.execute("kubeadm token list").stdout.split('\n')
for line in token_list:
if 'authentication,signing' in line:
print(crayons.white(line))
join_token = line.split()[0].strip()
join_url = f'{self.control_plane.apiserver_ip}:{self.control_plane.apiserver_port}'
else:
join_url = f'{self.instance.allowed_ip}:{self.control_plane.apiserver_port}'
join_token = self.instance.self_node_sudo.execute("kubeadm token list | awk '{print $1}'").stdout.split('TOKEN')[-1].strip()
while not join_token:
logging.warning(crayons.yellow('Join Token not found on master. Creating new join token...'))
self.instance.self_node_sudo.execute("kubeadm token create")
join_token = self.instance.self_node_sudo.execute("kubeadm token list | awk '{print $1}'").stdout.split('TOKEN')[-1].strip()
cert_hash = self.instance.self_node.execute("openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'").stdout.strip()
if not join_token or not cert_hash:
logging.error(crayons.red('Unable to retrieve join-token or cert-hash'))
return None
return (
f'kubeadm join {join_url} --token {join_token} --discovery-token-ca-cert-hash sha256:{cert_hash} --control-plane --certificate-key {certificate_key}'
) if control_plane_node else (
f'kubeadm join {join_url} --token {join_token} --discovery-token-ca-cert-hash sha256:{cert_hash}'
)
def apply_label_node(self, role, instance_name):
prepend = f'HOME={self.home}'
label_node = f'node-role.kubernetes.io/{role}='
labeled = self.local.run(f'{prepend} kubectl label nodes {instance_name} {label_node}', warn=True)
if labeled.ok:
print(crayons.green(f'Added label {label_node} to {instance_name}'))
else:
logging.warning(crayons.yellow(f'Label {label_node} already exists on {instance_name}'))
def get_current_context(self):
print(crayons.cyan('Verify that the cluster and context are the correct ones'))
current_context = self.local.run(f'HOME={self.home} kubectl config current-context').stdout.strip()
self.local.run(f'HOME={self.home} kubectl config view')
print(crayons.cyan('Are you in the correct cluster/context ? (y/N)'))
context_correct = input()
if context_correct not in ('Y', 'y'):
print(crayons.yellow('Aborting Operation'))
return None
return current_context
def helm_install_v2(self, patch=True, helm=True, tiller=True):
prepend = f'HOME={self.home}'
helm_script = 'https://git.io/get_helm.sh'
current_context = self.get_current_context()
if not current_context:
return
if helm:
print(crayons.cyan('Installing Helm locally'))
install = self.local.run(f'{prepend} curl -L {helm_script} | bash')
if not install.ok:
logging.error(crayons.red(f'Helm installation failed'))
return
self.local.run(f'echo "source <(helm completion bash)" >> {self.home}/.bashrc')
print(crayons.green('Helm installed locally'))
if tiller:
if not patch:
logging.warning(crayons.yellow('No-Patch (K8s versions > 1.16.*) installation is not implemented.'))
return
print(crayons.cyan('Bootstrapping Tiller with patch for K8s versions > 1.16.*'))
bootstrap = self.tiller_install_v2_patch()
if not bootstrap.ok:
logging.error(crayons.red(f'Helm initialization with Tiller failed'))
logging.warning(crayons.yellow('Rolling back installation'))
rollback = self.local.run(f'{prepend} helm reset --force --remove-helm-home')
if rollback.ok:
print(crayons.green('Rollback completed'))
return
tiller_ready = ''
while not tiller_ready:
print(crayons.white('Ping for tiller ready'))
tiller_ready = self.local.run(f'{prepend} kubectl get pod --namespace kube-system -l app=helm,name=tiller --field-selector=status.phase=Running').stdout.strip()
time.sleep(1)
print(crayons.green(f'Helm initialized with Tiller for context: {current_context}'))
self.wait_for_running_system_status()
time.sleep(10)
print(crayons.magenta('You might need to run "helm init --client-only to initialize repos"'))
self.local.run(f'{prepend} helm init --client-only')
def unset_local_cluster_config(self, cluster: 'ClusterAttributesSerializer'):
if not self.cluster_exists(cluster):
logging.warning(crayons.yellow(f'Cluster: {cluster.cluster.name} not found in config.'))
return
cluster_name = cluster.cluster.name
user = cluster.cluster.user
context = cluster.cluster.context
self.local.run(f'HOME={self.home} kubectl config use-context {context}')
self.local.run(f'HOME={self.home} kubectl config delete-cluster {cluster_name}')
self.local.run(f'HOME={self.home} kubectl config delete-context {context}')
self.local.run(f'HOME={self.home} kubectl config unset users.{user}')
def rollback_node(self):
logging.warning(crayons.yellow(f'Performing Node {self.instance.vm_attributes.name} Rollback.'))
rollback = self.instance.self_node_sudo.execute('kubeadm reset -f --v=5')
config_reset = self.instance.self_node.execute(f'rm -f $HOME/.kube/config', warn=True)
iptables_reset = self.instance.self_node_sudo.execute('su - root -c \'iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X\'')
if rollback.ok:
print(crayons.green('Rollback completed.'))
else:
logging.error(crayons.red('Rollback failed'))
return
if config_reset.ok:
print(crayons.green('Config removed.'))
else:
logging.warning(crayons.yellow('Config removal not performed.'))
if iptables_reset.ok:
print(crayons.green('IPTables reset completed.'))
else:
logging.error(crayons.red('IPTables reset failed.'))
return
def deploy_container_networking(self, cni_definitions: CNIDefinitions):
print(f'Deploying Container networking {self.control_plane.networking}')
if self.control_plane.networking == 'calico':
network = self.instance.self_node.execute(f'kubectl apply -f {os.path.join(self.remote_path, cni_definitions.file)}')
elif self.control_plane.networking in ('weave', 'weave-default'):
network = self.instance.self_node.execute(f'kubectl apply -f {cni_definitions.cni_url}')
else:
logging.warning(crayons.yellow(f'Networking {self.control_plane.networking} not supported currently.'))
return False
if network.ok:
print(crayons.green(f'Container Networking {self.control_plane.networking} deployed successfully.'))
return True
def check_install_prerequisites(self):
keepalived = LinuxPackage(command='keepalived', package='keepalived')
wget = LinuxPackage(command='wget', package='wget')
nc = LinuxPackage(command='nc', package='nmap-ncat')
ip = LinuxPackage(command='ip', package='iproute2')
curl = LinuxPackage(command='curl', package='curl')
for package in (wget, nc, ip, curl, keepalived):
package_exists = self.instance.self_node.execute(f'command {package.command} -h; echo $?', hide=True)
exit_code = package_exists.stdout.split()[-1].strip()
if exit_code != '0':
logging.warning(crayons.yellow(f'Package: {package.package} not found.'))
print(crayons.cyan(f'Installing {package.package}'))
self.instance.self_node_sudo.execute(f'yum install -y {package.package}')
def install_control_plane_loadbalancer(self, is_leader=False):
if not self.control_plane.ha_masters:
logging.warning(crayons.yellow('Skip install keepalived. Control Plane not Deployed in High Available Mode.'))
return None
local = LOCAL
host = self.instance.vm_attributes.name
remote_path = '/etc/keepalived'
state = 'MASTER' if is_leader else 'BACKUP'
priority = '101' if is_leader else '100'
interface = self.get_instance_interface()
virtual_ip = self.get_control_plane_virtual_ip()
script_file, local_script_path, script_error = self.generate_keepalived_healthcheck(virtual_ip)
config_file, local_config_path, config_error = self.generate_keepalived_config(
virtual_ip=virtual_ip,
interface=interface,
state=state,
priority=priority
)
if script_error or config_error:
logging.error(crayons.red(f'Abort keepalived install on {host}'))
return None
print(crayons.cyan(f'Sending config files to {host}'))
sent1 = local.run(f'scp {local_config_path} {host}:~')
sent2 = local.run(f'scp {local_script_path} {host}:~')
if sent1.ok and sent2.ok:
self.wait_pkg_lock()
print(crayons.blue(f'Installing keepalived service on {host}'))
self.check_install_prerequisites()
self.instance.self_node.execute(f'sudo mv {config_file} {remote_path} && sudo mv {script_file} {remote_path}')
self.instance.self_node_sudo.execute(f'chmod 0666 {remote_path}/{config_file}')
restart = self.instance.self_node_sudo.execute(f'systemctl restart keepalived')
if restart.ok:
self.instance.self_node_sudo.execute('systemctl status keepalived')
test_connection = f'if nc -v -w 5 {virtual_ip} {self.control_plane.apiserver_port}; then echo "Success"; fi'
output = self.instance.self_node.execute(test_connection).stderr
if 'Connection refused' in output.strip():
print(crayons.green(f'Keepalived running on {host} with virtual ip: {virtual_ip}'))
return virtual_ip
