def installWeaveNodeAfter(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
weaveConfigPath = self.getLocalPath('weaveConfigPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
tmpPath = self.getLocalPath('tmpPath')
cmdRemote = remoteBinPath + "/kubectl delete -f " + remotePluginPath + "/weave/weave.yaml"
robj_admin.sudo(cmdRemote)
#upload yaml file
ips_data = self.config_object.get_node_by_attr("name", "ips")
podClusterIpRange = ips_data.find('podClusterIpRange').text.replace(
"/", "\/")
sedRegex = "sed \"s/{podClusterIpRange}/%s/g\"" % podClusterIpRange
file_tmp = tmpPath + "/weave.yaml"
file_config = weaveConfigPath + "/weave.yaml"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_admin.local(cmd_local)
# 上传文件到远程主机
robj_admin.upload(file_tmp, remotePluginPath + "/weave/", True)
#install weave plugin
# yamlWeave="https://cloud.weave.works/k8s/net?k8s-version=\$(${remoteBinPath}/kubectl version | base64 | tr -d '\\\n')"
cmdRemote = remoteBinPath + "/kubectl apply -f " + remotePluginPath + "/weave/weave.yaml"
robj_admin.sudo(cmdRemote)
def installCniNodeNormal(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj_node = Remote(node_env)
cniBinPath = self.getLocalPath('cniBinPath')
cniConfigPath = self.getLocalPath('cniConfigPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
tmpPath = self.getLocalPath('tmpPath')
remoteCniBinPath = self.getRemotePath('remoteCniBinPath')
remoteCniCfgPath = self.getRemotePath('remoteCniCfgPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
#stop relative service if the service is running
cmdRemote = "systemctl stop kube-kubelet"
robj_node.sudo(cmdRemote)
# upload cni files
robj_node.upload(cniBinPath + "/loopback", remoteCniBinPath + "/")
robj_node.upload(cniBinPath + "/portmap", remoteCniBinPath + "/")
robj_node.upload(cniConfigPath + "/99-loopback.conf",
remoteCniCfgPath + "/")
#upload kubelet service config file for cni
file_config = kbsConfigPath + "/kube-kubelet.conf"
file_tmp = tmpPath + "/kube-kubelet.conf"
plugin_name = "cni"
sedRegex = "sed \"s/^--hostname-override={nodeName}/--hostname-override=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/^--network-plugin={pluginName}/--network-plugin=%s/g\"" % plugin_name
sedRegex = sedRegex + " | sed \"s/^--cni-conf-dir={confDir}/--cni-conf-dir=%s/g\"" % remoteCniCfgPath.replace(
"/", "\/")
sedRegex = sedRegex + " | sed \"s/^--cni-bin-dir={binDir}/--cni-bin-dir=%s/g\"" % remoteCniBinPath.replace(
"/", "\/")
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
print(cmd_local)
robj_node.local(cmd_local)
# 上传文件到远程主机
robj_node.upload(file_tmp, remoteCfgPath + "/", True)
print("systemd daemon reload ...")
cmdRemote = "systemctl daemon-reload"
robj_node.sudo(cmdRemote)
print("restart kubelet service ...")
cmdRemote = "systemctl restart kube-kubelet.service"
robj_node.sudo(cmdRemote)
print("check kubelet service ...")
cmdRemote = "systemctl status kube-kubelet.service"
robj_node.sudo(cmdRemote)
def installDockerPrivate(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
docker_data = self.config_object.get_node_by_attr("name", "docker")
dockerUsername = docker_data.find('username').text
dockerPassword = docker_data.find('password').text
dockerDomain = docker_data.find('domain').text
tmpPath = self.getLocalPath('tmpPath')
dockerConfigPath = self.getLocalPath('dockerConfigPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
#login docker registry
cmdRemote = "docker login --username=%s --password=%s %s" % (
dockerUsername, dockerPassword, dockerDomain)
robj_admin.sudo(cmdRemote)
#download docker login config file form master1 to local
robj_admin.download("/root/.docker/config.json", tmpPath + "/")
#view docker login config file content in local, and generate login credentials
# dockerSecret = $(cat ./config.json | base64 -w 0)
file_config = dockerConfigPath + "/registry-pull-secret.yaml"
file_tmp = tmpPath + "/registry-pull-secret.yaml"
with open(tmpPath + '/config.json') as file_object:
contents = file_object.read()
dockerSecret = base64.b64encode(
contents.rstrip().encode("utf-8")).decode("utf-8")
print("dockerSecret: " + dockerSecret)
sedRegex = "s/{dockerSecret}/%s/g" % dockerSecret.replace(
"/", r"\/")
cmdLocal = "cat %s | sed \"%s\" > %s" % (file_config, sedRegex,
file_tmp)
print(cmdLocal)
robj_admin.local(cmdLocal)
robj_admin.upload(file_tmp, remoteCfgPath + "/", True)
#create docker secret pod
cmdRemote = remoteBinPath + "/kubectl create -f " + remoteCfgPath + "/registry-pull-secret.yaml"
robj_admin.sudo(cmdRemote)
#get docker secret
cmdRemote = remoteBinPath + "/kubectl get secret"
robj_admin.sudo(cmdRemote)
def installCoredns(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
tmpPath = self.getLocalPath('tmpPath')
corednsConfigPath = self.getLocalPath('corednsConfigPath')
#upload
robj_admin.upload(corednsConfigPath + "/coredns-rbac.yaml",
remotePluginPath + "/coredns/", True)
robj_admin.upload(corednsConfigPath + "/coredns-sa.yaml",
remotePluginPath + "/coredns/", True)
robj_admin.upload(corednsConfigPath + "/coredns-configmap.yaml",
remotePluginPath + "/coredns/", True)
robj_admin.upload(corednsConfigPath + "/coredns-deployment.yaml",
remotePluginPath + "/coredns/", True)
ips_data = self.config_object.get_node_by_attr("name", "ips")
serviceClusterDnsIp = ips_data.find('serviceClusterDnsIp').text
sedRegex = "sed \"s/{serviceClusterDnsIp}/%s/g\"" % serviceClusterDnsIp
file_tmp = tmpPath + "/coredns-service.yaml"
file_config = corednsConfigPath + "/coredns-service.yaml"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_admin.local(cmd_local)
# 上传文件到远程主机
robj_admin.upload(file_tmp, remotePluginPath + "/coredns/", True)
#install
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-rbac.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-sa.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-configmap.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-deployment.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-service.yaml"
robj_admin.sudo(cmdRemote)
def cleanup(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj_node = Remote(node_env)
remoteEtcdDataPath = self.getRemotePath('remoteEtcdDataPath')
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
tmpPath = self.getLocalPath('tmpPath')
cmdRemote = "systemctl stop kube-kubelet"
robj_node.sudo(cmdRemote)
cmdRemote = "systemctl stop kube-proxy"
robj_node.sudo(cmdRemote)
cmdRemote = "systemctl stop kube-apiserver"
robj_node.sudo(cmdRemote)
cmdRemote = "systemctl stop kube-controller-manager"
robj_node.sudo(cmdRemote)
cmdRemote = "systemctl stop kube-scheduler"
robj_node.sudo(cmdRemote)
cmdRemote = "systemctl stop docker"
robj_node.sudo(cmdRemote)
robj_node.cleanup(remoteEtcdDataPath)
print("cache files of etcd is removed ...")
robj_node.sudo("ls -l " + remoteEtcdDataPath + "/")
res = robj_node.cleanup(remoteSslPath)
if (res == False):
print("failed to cleanup " + remoteSslPath)
print("ssl files is removed ...")
robj_node.sudo("ls -l " + remoteSslPath + "/")
robj_node.cleanup(remoteCfgPath)
print("config files is removed ...")
robj_node.sudo("ls -l " + remoteCfgPath + "/")
robj_node.cleanupLocal(tmpPath, True)
print("tmp files is removed ...")
robj_node.local("ls -l " + tmpPath + "/")
def installDemoDockerPrivate(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
docker_data = self.config_object.get_node_by_attr("name", "docker")
dockerUsername = docker_data.find('username').text
dockerPassword = docker_data.find('password').text
dockerDomain = docker_data.find('domain').text
remoteBinPath = self.getRemotePath('remoteBinPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
dockerConfigPath = self.getLocalPath('dockerConfigPath')
#get docker secret
cmdRemote = remoteBinPath + "/kubectl get secret"
robj_admin.sudo(cmdRemote)
#valid
busyboxAddress = "registry.cn-beijing.aliyuncs.com/ducafe/busybox"
cmdLocal = "docker images | grep %s | awk '{print $3}'" % busyboxAddress
robj_admin.local(cmdLocal)
imageId = robj_admin.getResult().stdout.rstrip()
# print(robj_admin.getResult().exited)
if imageId == "":
print("upload busybox image to aliyun repository ...")
cmdLocal = "docker pull registry.cn-beijing.aliyuncs.com/ducafe/busybox:1.24"
robj_admin.local(cmdLocal)
cmdLocal = "docker images | grep %s | awk '{print $3}'" % busyboxAddress
robj_admin.local(cmdLocal)
imageId = robj_admin.getResult().stdout.rstrip()
#login docker registry
cmdLocal = "docker login --username=%s --password=%s %s" % (dockerUsername, dockerPassword, dockerDomain)
robj_admin.local(cmdLocal)
cmdLocal = "docker tag %s %s/kube-systems/busybox-demo:1.24" % (imageId, dockerDomain)
robj_admin.local(cmdLocal)
cmdLocal = "docker images"
robj_admin.local(cmdLocal)
cmdLocal = "docker push %s/kube-systems/busybox-demo:1.24" % dockerDomain
robj_admin.local(cmdLocal)
#upload
robj_admin.upload(dockerConfigPath+"/busybox-demo.yaml", remotePluginPath+"/docker/")
#install
cmdRemote = remoteBinPath + "/kubectl create -f "+remotePluginPath+"/docker/busybox-demo.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = "sleep 2s"
robj_admin.sudo(cmdRemote)
def installKbsKubelet(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
print("node ip: " + node_env.host)
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
print("master ip: " + node_env1.host)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
tmpPath = self.getLocalPath('tmpPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
#download ca files from master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download etcd ca files from master1 to local
robj_master1.download(remoteSslPath + "/etcd.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd-key.pem", tmpPath + "/")
robj_master1.download(remoteCfgPath + "/token.csv", tmpPath + "/")
#download kubeconfig files from admin to local
robj_admin.download(remoteCfgPath + "/bootstrap.kubeconfig",
tmpPath + "/")
#stop relative service if the service is running
cmdRemote = "systemctl stop kube-kubelet"
robj.sudo(cmdRemote)
#upload binary file for kube processes
if robj.checkpath(remoteBinPath + "/kubelet") == False:
robj.upload(kbsBinPath + "/kubelet", remoteBinPath + "/")
#upload ca files to node1
robj.upload(tmpPath + "/ca.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/ca-key.pem", remoteSslPath + "/")
#upload etcd ca files to node1
robj.upload(tmpPath + "/etcd.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd-key.pem", remoteSslPath + "/")
#upload config files to node1
robj.upload(tmpPath + "/bootstrap.kubeconfig", remoteCfgPath + "/")
robj.upload(tmpPath + "/token.csv", remoteCfgPath + "/")
sedRegex = "sed \"s/{serviceClusterDnsIp}/%s/g\"" % self.serviceClusterDnsIp
file_tmp = tmpPath + "/kubelet.config"
file_config = kbsConfigPath + "/kubelet.config"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/")
#upload systemd service files to node
robj.upload(kbsConfigPath + "/kube-kubelet.service",
remoteSystemdPath + "/")
print("node ip: " + node_env.host)
#upload kubelet service config file
file_config = kbsConfigPath + "/kube-kubelet.conf"
file_tmp = tmpPath + "/kube-kubelet.conf"
sedRegex = "sed \"s/^--hostname-override={nodeName}/--hostname-override=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/^--network-plugin={pluginName}//g\""
sedRegex = sedRegex + " | sed \"s/^--cni-conf-dir={confDir}//g\""
sedRegex = sedRegex + " | sed \"s/^--cni-bin-dir={binDir}//g\""
sedRegex = sedRegex + " | sed \"s/{serviceClusterDnsIp}/%s/g\"" % self.serviceClusterDnsIp
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath, True)
print("systemd daemon reload ...")
cmdRemote = "systemctl daemon-reload"
robj.sudo(cmdRemote)
print("enable kubelet service ...")
cmdRemote = "systemctl enable kube-kubelet.service"
robj.sudo(cmdRemote)
print("restart kubelet service ...")
cmdRemote = "systemctl restart kube-kubelet.service"
robj.sudo(cmdRemote)
print("check kubelet service ...")
cmdRemote = "systemctl status kube-kubelet.service"
robj.sudo(cmdRemote)
print("sleep 5 seconds ...")
cmdRemote = "sleep 5"
robj.sudo(cmdRemote)
print("restart kube apiserver ...")
cmdRemote = "systemctl restart kube-apiserver"
robj_master1.sudo(cmdRemote)
print("show csr info: ")
cmdRemote = remoteBinPath + "/kubectl get csr"
robj_admin.sudo(cmdRemote)
print("pass tls request: ")
cmdRemote = remoteBinPath + "/kubectl get csr | grep 'Pending'"
cmdRemote = cmdRemote + " | awk 'NR>0 {print $1}'"
cmdRemote = cmdRemote + " | xargs " + remoteBinPath + "/kubectl certificate approve"
robj_admin.sudo(cmdRemote)
def installKbsProxy(self, nodename):
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
tmpPath = self.getLocalPath('tmpPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
#download proxy cert files from master1
robj_master1.download(remoteSslPath + "/kube-proxy.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kube-proxy.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kube-proxy-key.pem",
tmpPath + "/")
#download proxy kubeconfig files from admin node
robj_admin.download(remoteCfgPath + "/kube-proxy.kubeconfig",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-ca.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-ca-key.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.csr",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client-key.pem",
tmpPath + "/")
robj.upload(tmpPath + "/front-proxy-ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-ca-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client-key.pem",
remoteSslPath + "/")
#stop relative service if the service is running
cmdRemote = "systemctl stop kube-proxy"
robj.sudo(cmdRemote)
#install some packages for proxy
cmdRemote = "yum install -y ipvsadm ipset conntrack"
robj.sudo(cmdRemote)
#upload binary file for kube processes
if robj.checkpath(remoteBinPath + "/kube-proxy") == False:
robj.upload(kbsBinPath + "/kube-proxy", remoteBinPath + "/")
#upload proxy cert files to node1
robj.upload(tmpPath + "/kube-proxy.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/kube-proxy-key.pem", remoteSslPath + "/")
#upload proxy kubeconfig files to node1
robj.upload(tmpPath + "/kube-proxy.kubeconfig", remoteCfgPath + "/")
#upload systemd service files to node1
robj.upload(kbsConfigPath + "/kube-proxy.service",
remoteSystemdPath + "/")
#upload proxy service config file
sedRegex = "sed \"s/^--hostname-override={nodeName}/--hostname-override=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/{podClusterIpRange}/%s/g\"" % self.podClusterIpRange
file_tmp = tmpPath + "/kube-proxy.conf"
file_config = kbsConfigPath + "/kube-proxy.conf"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/", True)
print("systemd daemon reload ...")
cmdRemote = "systemctl daemon-reload"
robj.sudo(cmdRemote)
print("enable proxy service ...")
cmdRemote = "systemctl enable kube-proxy.service"
robj.sudo(cmdRemote)
print("restart proxy service ...")
cmdRemote = "systemctl restart kube-proxy.service"
robj.sudo(cmdRemote)
print("check proxy service ...")
cmdRemote = "systemctl status kube-proxy.service"
robj.sudo(cmdRemote)
def installKbsMasterBefore(self):
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
tmpPath = self.getLocalPath('tmpPath')
etcdConfigPath = self.getLocalPath('etcdConfigPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
#download ca files form master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
# robj_master1.download(remoteSslPath+"/ca.pem", tmpPath+"/")
# robj_master1.download(remoteSslPath+"/ca-key.pem", tmpPath+"/")
#upload ca cert files
# robj_master1.upload(tmpPath+"/ca.pem", remoteSslPath+"/")
# robj_master1.upload(tmpPath+"/ca-key.pem", remoteSslPath+"/")
#upload files
robj_master1.upload(etcdConfigPath + "/ca-config.json",
remoteSslPath + "/")
# upload kubernetes csr config file
file_tmp = tmpPath + "/kubernetes-csr.json"
file_config = kbsConfigPath + "/kubernetes-csr.json"
sedRegex = "sed \"s/{kbsMasterIp1}/%s/g\"" % self.getEnv(
"master1").host
sedRegex = sedRegex + " | sed \"s/{kbsMasterIp2}/%s/g\"" % self.getEnv(
"master2").host
sedRegex = sedRegex + " | sed \"s/{kbsMasterIp3}/%s/g\"" % self.getEnv(
"master3").host
sedRegex = sedRegex + " | sed \"s/{kbsMasterProxyIp}/%s/g\"" % self.proxy_host
sedRegex = sedRegex + " | sed \"s/{serviceClusterGatewayIp}/%s/g\"" % self.serviceClusterGatewayIp
sedRegex = sedRegex + " | sed \"s/{workerNodeIp}/%s/g\"" % self.getEnv(
"node1").host
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_master1.local(cmd_local)
# 上传文件到远程主机
robj_master1.upload(file_tmp, remoteSslPath + "/")
# robj_master1.upload(kbsConfigPath+"/admin-csr.json", remoteSslPath+"/")
#create token.csv file for bootstrap authority in master1
print("ready to create tocken.csv file in master1 ...")
cmdRemote = "/bin/sh -c \'BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d \" \")"
cmdRemote = cmdRemote + "; BOOTSTRAP_CONTENT=\"${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,\\\"system:kubelet-bootstrap\\\"\""
cmdRemote = cmdRemote + "; echo ${BOOTSTRAP_CONTENT} > " + remoteCfgPath + "/token.csv\'"
# print(cmdRemote)
robj_master1.sudo(cmdRemote)
robj_master1.sudo("cat " + remoteCfgPath + "/token.csv")
#create kubernetes cert files in master1
print("ready to create kubernetes cert file in master1 ...")
cmdRemote = remoteLocalBinPath + "/cfssl gencert"
cmdRemote = cmdRemote + " -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/kubernetes-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/kubernetes"
robj_master1.sudo(cmdRemote)
#create metrics api aggregator cert files in master1
robj_master1.upload(kbsConfigPath + "/front-proxy-ca-csr.json",
remoteSslPath + "/")
robj_master1.upload(kbsConfigPath + "/front-proxy-client-csr.json",
remoteSslPath + "/", True)
# generate ca root cert
cmdRemote = remoteLocalBinPath + "/cfssl gencert -initca " + remoteSslPath + "/front-proxy-ca-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/front-proxy-ca"
robj_master1.sudo(cmdRemote)
# generate client cert for each node
cmdRemote = remoteLocalBinPath + "/cfssl gencert -ca=" + remoteSslPath + "/front-proxy-ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/front-proxy-ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/front-proxy-client-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/front-proxy-client"
robj_master1.sudo(cmdRemote)
def installKbsNodeBefore(self):
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
tmpPath = self.getLocalPath('tmpPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
#download token csv file from master1 to local
robj_master1.download(remoteCfgPath + "/token.csv", tmpPath + "/")
#####for kubelet#####
#delete old role binding in master1
cmdRemote = remoteBinPath + "/kubectl delete clusterrolebinding/kubelet-bootstrap"
robj_admin.sudo(cmdRemote)
#create role binding in master1
cmdRemote = remoteBinPath + "/kubectl create clusterrolebinding kubelet-bootstrap"
cmdRemote = cmdRemote + " --clusterrole=system:node-bootstrapper"
cmdRemote = cmdRemote + " --user=kubelet-bootstrap"
robj_admin.sudo(cmdRemote)
#set cluster parameters and create kubelet bootstrapping kubeconfig file
cmdRemote = remoteBinPath + "/kubectl config set-cluster kubernetes"
cmdRemote = cmdRemote + " --certificate-authority=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --server=https://" + self.proxy_host + ":" + self.proxy_port
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
robj_admin.local("sleep 3")
f = open(tmpPath + '/token.csv', 'r')
csv_file = csv.reader(f)
csv_content = next(csv_file)
csv_token = csv_content[0]
f.close()
#create authority parameters for client
cmdRemote = remoteBinPath + "/kubectl config set-credentials kubelet-bootstrap"
cmdRemote = cmdRemote + " --token=" + csv_token
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
#set context parameters
cmdRemote = remoteBinPath + "/kubectl config set-context kubernetes"
cmdRemote = cmdRemote + " --cluster=kubernetes"
cmdRemote = cmdRemote + " --user=kubelet-bootstrap"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
#set default context
cmdRemote = remoteBinPath + "/kubectl config use-context kubernetes"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
#####for proxy#####
#upload proxy csr file to master1
robj_master1.upload(kbsConfigPath + "/kube-proxy-csr.json",
remoteSslPath + "/")
#delete old proxy cert files in master1
robj_master1.rmfile(remoteSslPath + "/kube-proxy.csr")
robj_master1.rmfile(remoteSslPath + "/kube-proxy.pem")
robj_master1.rmfile(remoteSslPath + "/kube-proxy-key.pem")
#create proxy cert files in master1
cmdRemote = "/usr/local/bin/cfssl gencert"
cmdRemote = cmdRemote + " -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/kube-proxy-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/kube-proxy"
robj_master1.sudo(cmdRemote)
#download cert files from master1 to local
robj_master1.download(remoteSslPath + "/kube-proxy.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kube-proxy-key.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
#upload cert files from local to admin node
robj_admin.upload(tmpPath + "/kube-proxy.pem", remoteSslPath + "/")
robj_admin.upload(tmpPath + "/kube-proxy-key.pem", remoteSslPath + "/")
robj_admin.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
#create kube-proxy kubeconfig file in admin node
cmdRemote = remoteBinPath + "/kubectl config set-cluster kubernetes"
cmdRemote = cmdRemote + " --certificate-authority=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --server=https://" + self.proxy_host + ":" + self.proxy_port
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl config set-credentials kube-proxy"
cmdRemote = cmdRemote + " --client-certificate=" + remoteSslPath + "/kube-proxy.pem"
cmdRemote = cmdRemote + " --client-key=" + remoteSslPath + "/kube-proxy-key.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl config set-context kubernetes"
cmdRemote = cmdRemote + " --cluster=kubernetes"
cmdRemote = cmdRemote + " --user=kube-proxy"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl config use-context kubernetes"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
def installKbsMasterNormal(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
tmpPath = self.getLocalPath('tmpPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
#stop relative service if the service is running
robj.sudo("systemctl stop kube-apiserver")
robj.sudo("systemctl stop kube-controller-manager")
robj.sudo("systemctl stop kube-scheduler")
#install some packages for proxy
cmdRemote = "yum install -y ipvsadm ipset conntrack"
robj.sudo(cmdRemote)
#download ca files from master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download etcd ca files from master1 to local
robj_master1.download(remoteSslPath + "/etcd.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd-key.pem", tmpPath + "/")
#download kubernetes ca files from master1 to local
robj_master1.download(remoteSslPath + "/kubernetes.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kubernetes.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kubernetes-key.pem",
tmpPath + "/")
#download metrics api ca files from master1 to local
robj_master1.download(remoteSslPath + "/front-proxy-ca.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.csr",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client-key.pem",
tmpPath + "/")
#download token csv file from master1 to local
robj_master1.download(remoteCfgPath + "/token.csv", tmpPath + "/")
#upload ca files to master node
robj.upload(tmpPath + "/ca.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/ca-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/kubernetes.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/kubernetes.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/kubernetes-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client-key.pem",
remoteSslPath + "/")
robj.upload(tmpPath + "/token.csv", remoteCfgPath + "/")
#upload binary files to master node
robj.upload((kbsBinPath + "/kube-apiserver"), (remoteBinPath + "/"))
robj.upload((kbsBinPath + "/kube-controller-manager"),
(remoteBinPath + "/"))
robj.upload((kbsBinPath + "/kube-scheduler"), (remoteBinPath + "/"))
#upload kubernetes api server config file
file_tmp = tmpPath + "/kube-apiserver.conf"
file_config = kbsConfigPath + "/kube-apiserver.conf"
sedRegex = "sed \"s/^--advertise-address={nodeIp}/--advertise-address=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp1}/%s/g\"" % self.getEnv(
"master1").host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp2}/%s/g\"" % self.getEnv(
"master2").host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp3}/%s/g\"" % self.getEnv(
"master3").host
sedRegex = sedRegex + " | sed \"s/{serviceClusterIpRange}/%s/g\"" % self.serviceClusterIpRange
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/", True)
# upload kubernetes controller manager config file
file_tmp = tmpPath + "/kube-controller-manager.conf"
file_config = kbsConfigPath + "/kube-controller-manager.conf"
sedRegex = "sed \"s/{serviceClusterIpRange}/%s/g\"" % self.serviceClusterIpRange
sedRegex = sedRegex + " | sed \"s/{podClusterIpRange}/%s/g\"" % self.podClusterIpRange
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/", True)
# upload kubernetes scheduler config file
robj.upload(kbsConfigPath + "/kube-scheduler.conf",
remoteCfgPath + "/", True)
#upload systemd service file to master node
robj.upload(kbsConfigPath + "/kube-apiserver.service",
remoteSystemdPath + "/", True)
robj.upload(kbsConfigPath + "/kube-controller-manager.service",
remoteSystemdPath + "/", True)
robj.upload(kbsConfigPath + "/kube-scheduler.service",
remoteSystemdPath + "/", True)
robj.sudo("systemctl daemon-reload")
robj.sudo("systemctl enable kube-apiserver")
robj.sudo("systemctl restart kube-apiserver")
robj.sudo("systemctl status kube-apiserver")
robj.sudo("systemctl enable kube-controller-manager")
robj.sudo("systemctl restart kube-controller-manager")
robj.sudo("systemctl status kube-controller-manager")
robj.sudo("systemctl enable kube-scheduler")
robj.sudo("systemctl restart kube-scheduler")
robj.sudo("systemctl status kube-scheduler")
def installEtcdNodeNormal(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
remoteEtcdDataPath = self.getRemotePath('remoteEtcdDataPath')
etcdBinPath = self.getLocalPath('etcdBinPath')
etcdConfigPath = self.getLocalPath('etcdConfigPath')
tmpPath = self.getLocalPath('tmpPath')
#upload ca files
robj.upload(tmpPath + "/ca.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/ca-key.pem", remoteSslPath + "/")
#upload etcd ca files
robj.upload(tmpPath + "/etcd.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd-key.pem", remoteSslPath + "/")
#upload binary file for etcd processes
if robj.checkpath((remoteBinPath + "/etcd")) == False:
robj.upload(etcdBinPath + "/etcd", remoteBinPath + "/")
if robj.checkpath((remoteBinPath + "/etcdctl")) == False:
robj.upload(etcdBinPath + "/etcdctl", remoteBinPath + "/")
#upload etcd config file
file_config = etcdConfigPath + "/etcd.conf"
file_tmp = tmpPath + "/etcd.conf"
sedRegex = "sed \"s/{etcdNodeName}/%s/g\"" % ("etcd-" + nodename)
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp}/%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeName1}/%s/g\"" % ("etcd-" +
"master1")
node1_env = self.getEnv("master1")
etcdNodeIp1 = node1_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp1}/%s/g\"" % etcdNodeIp1
sedRegex = sedRegex + " | sed \"s/{etcdNodeName2}/%s/g\"" % ("etcd-" +
"master2")
node2_env = self.getEnv("master2")
etcdNodeIp2 = node2_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp2}/%s/g\"" % etcdNodeIp2
sedRegex = sedRegex + " | sed \"s/{etcdNodeName3}/%s/g\"" % ("etcd-" +
"master3")
node3_env = self.getEnv("master3")
etcdNodeIp3 = node3_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp3}/%s/g\"" % etcdNodeIp3
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
robj.upload(file_tmp, remoteCfgPath + "/")
robj.upload(etcdConfigPath + "/etcd.service", remoteSystemdPath + "/")
if (remoteEtcdDataPath != "" and remoteEtcdDataPath != "/"
and robj.checkpath(remoteEtcdDataPath)):
robj.rmfile(remoteEtcdDataPath + "/*")
print("cache files of etcd is removed ...")
robj.sudo("ls -l " + remoteEtcdDataPath + "/")
else:
robj.mkdir(remoteEtcdDataPath)
#systemd for etcd process
robj.sudo("systemctl daemon-reload")
robj.sudo("systemctl enable etcd.service")
cmdRemote = "nohup systemctl restart etcd.service &> /dev/null &"
robj.sudo(cmdRemote)
def installEtcdNodePrepare(self):
#stop relative service if the service is running
print("stop relative service if the service is running ...")
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
print("stop etcd service in master1")
robj_master1.sudo("systemctl stop etcd")
node_env2 = self.getEnv("master2")
# 初始化远程工具对象
robj_master2 = Remote(node_env2)
print("stop etcd service in master2")
robj_master2.sudo("systemctl stop etcd")
node_env3 = self.getEnv("master3")
# 初始化远程工具对象
robj_master3 = Remote(node_env3)
print("stop etcd service in master3")
robj_master3.sudo("systemctl stop etcd")
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
etcdConfigPath = self.getLocalPath('etcdConfigPath')
tmpPath = self.getLocalPath('tmpPath')
#upload ca cert config file
robj_master1.upload(etcdConfigPath + "/ca-csr.json",
remoteSslPath + "/")
robj_master1.upload(etcdConfigPath + "/ca-config.json",
remoteSslPath + "/")
#upload etcd csr file
file_config = etcdConfigPath + "/etcd-csr.json"
file_tmp = tmpPath + "/etcd-csr.json"
node1_env = self.getEnv("master1")
etcdNodeIp1 = node1_env.host
sedRegex = "sed \"s/{etcdNodeIp1}/%s/g\"" % etcdNodeIp1
node2_env = self.getEnv("master2")
etcdNodeIp2 = node2_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp2}/%s/g\"" % etcdNodeIp2
node3_env = self.getEnv("master3")
etcdNodeIp3 = node3_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp3}/%s/g\"" % etcdNodeIp3
node4_env = self.getEnv("node1")
workerNodeIp = node4_env.host
sedRegex = sedRegex + " | sed \"s/{workerNodeIp}/%s/g\"" % workerNodeIp
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_master1.local(cmd_local)
robj_master1.upload(file_tmp, remoteSslPath + "/")
# generate ca root cert
cmdRemote = remoteLocalBinPath + "/cfssl gencert -initca " + remoteSslPath + "/ca-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/ca"
robj_master1.sudo(cmdRemote)
#generate etcd cert for each node
cmdRemote = remoteLocalBinPath + "/cfssl gencert -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/etcd-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/etcd"
robj_master1.sudo(cmdRemote)
#download ca files form master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download etcd ca files form master1 to local
robj_master1.download(remoteSslPath + "/etcd.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd-key.pem", tmpPath + "/")