def installWeaveNodeAfter(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
weaveConfigPath = self.getLocalPath('weaveConfigPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
tmpPath = self.getLocalPath('tmpPath')
cmdRemote = remoteBinPath + "/kubectl delete -f " + remotePluginPath + "/weave/weave.yaml"
robj_admin.sudo(cmdRemote)
#upload yaml file
ips_data = self.config_object.get_node_by_attr("name", "ips")
podClusterIpRange = ips_data.find('podClusterIpRange').text.replace(
"/", "\/")
sedRegex = "sed \"s/{podClusterIpRange}/%s/g\"" % podClusterIpRange
file_tmp = tmpPath + "/weave.yaml"
file_config = weaveConfigPath + "/weave.yaml"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_admin.local(cmd_local)
# 上传文件到远程主机
robj_admin.upload(file_tmp, remotePluginPath + "/weave/", True)
#install weave plugin
# yamlWeave="https://cloud.weave.works/k8s/net?k8s-version=\$(${remoteBinPath}/kubectl version | base64 | tr -d '\\\n')"
cmdRemote = remoteBinPath + "/kubectl apply -f " + remotePluginPath + "/weave/weave.yaml"
robj_admin.sudo(cmdRemote)
def installDemoBusybox(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
# tmpPath = self.getLocalPath('tmpPath')
corednsConfigPath = self.getLocalPath('corednsConfigPath')
#uninstall old service
print("remove busybox container ...")
cmdRemote = remoteBinPath + "/kubectl delete -f "+remotePluginPath+"/coredns/busybox.yaml"
robj_admin.sudo(cmdRemote)
#upload
print("upload busybox yaml ...")
robj_admin.upload(corednsConfigPath+"/busybox.yaml", remotePluginPath+"/coredns/")
#install
print("create busybox container ...")
cmdRemote = remoteBinPath + "/kubectl create -f "+remotePluginPath+"/coredns/busybox.yaml"
robj_admin.sudo(cmdRemote)
print("sleep 5 seconds ...")
robj_admin.sudo("sleep 5s")
#view
print("list pods of busybox ...")
cmdRemote = remoteBinPath + "/kubectl get pods -o wide"
robj_admin.sudo(cmdRemote)
def installDemoIngressHttps(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSslPath = self.getRemotePath('remoteSslPath')
ingressConfigPath = self.getLocalPath('ingressConfigPath')
# tls support
domainName = "sslexample.foo.com"
secretName = domainName.replace(".", "-")
print("generate tls key file ...")
keyFile = remoteSslPath + "/tls-ingress-"+secretName+".key"
certFile = remoteSslPath + "/tls-ingress-"+secretName+".crt"
#generate tls key
cmdRemote = "openssl genrsa -out "+keyFile+" 2048"
robj_admin.sudo(cmdRemote)
#generate tls cert
print("generate tls cert file ...")
cmdRemote = "openssl req -new -x509 -key "+keyFile+" -out "+certFile
cmdRemote = cmdRemote + " -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN="+domainName
robj_admin.sudo(cmdRemote)
#generate secret
print("generate secret ...")
cmdRemote = remoteBinPath + "/kubectl create secret tls "+secretName
cmdRemote = cmdRemote + " --cert="+certFile+" --key="+keyFile
robj_admin.sudo(cmdRemote)
#get secret
print("get secret ...")
cmdRemote = remoteBinPath + "/kubectl get secret"
robj_admin.sudo(cmdRemote)
#describe secret
print("describe secret ...")
cmdRemote = remoteBinPath + "/kubectl describe secret "+secretName
robj_admin.sudo(cmdRemote)
# remove ingress rules of https
print("remove ingress rules of https ...")
cmdRemote = remoteBinPath + "/kubectl delete -f "
cmdRemote = cmdRemote + remotePluginPath + "/ingress/nginx-ingress-rules-https.yaml"
robj_admin.sudo(cmdRemote)
#upload ingress rules of https
print("upload ingress rules yaml ...")
robj_admin.upload(ingressConfigPath+"/nginx-ingress-rules-https.yaml", remotePluginPath+"/ingress/")
#install ingress rules of https
print("install ingress rules ...")
cmdRemote = remoteBinPath + "/kubectl apply -f "
cmdRemote = cmdRemote + remotePluginPath+"/ingress/nginx-ingress-rules-https.yaml"
robj_admin.sudo(cmdRemote)
def upload(c, dirlocal, dirdes, nodename):
print(dirlocal)
print(nodename)
# 初始化远程工具对象
robj = Remote(getEnv(nodename))
# 上传文件到远程主机
robj.upload(dirlocal, dirdes)
print("Upload Success.")
def installDemoDockerPrivate(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
docker_data = self.config_object.get_node_by_attr("name", "docker")
dockerUsername = docker_data.find('username').text
dockerPassword = docker_data.find('password').text
dockerDomain = docker_data.find('domain').text
remoteBinPath = self.getRemotePath('remoteBinPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
dockerConfigPath = self.getLocalPath('dockerConfigPath')
#get docker secret
cmdRemote = remoteBinPath + "/kubectl get secret"
robj_admin.sudo(cmdRemote)
#valid
busyboxAddress = "registry.cn-beijing.aliyuncs.com/ducafe/busybox"
cmdLocal = "docker images | grep %s | awk '{print $3}'" % busyboxAddress
robj_admin.local(cmdLocal)
imageId = robj_admin.getResult().stdout.rstrip()
# print(robj_admin.getResult().exited)
if imageId == "":
print("upload busybox image to aliyun repository ...")
cmdLocal = "docker pull registry.cn-beijing.aliyuncs.com/ducafe/busybox:1.24"
robj_admin.local(cmdLocal)
cmdLocal = "docker images | grep %s | awk '{print $3}'" % busyboxAddress
robj_admin.local(cmdLocal)
imageId = robj_admin.getResult().stdout.rstrip()
#login docker registry
cmdLocal = "docker login --username=%s --password=%s %s" % (dockerUsername, dockerPassword, dockerDomain)
robj_admin.local(cmdLocal)
cmdLocal = "docker tag %s %s/kube-systems/busybox-demo:1.24" % (imageId, dockerDomain)
robj_admin.local(cmdLocal)
cmdLocal = "docker images"
robj_admin.local(cmdLocal)
cmdLocal = "docker push %s/kube-systems/busybox-demo:1.24" % dockerDomain
robj_admin.local(cmdLocal)
#upload
robj_admin.upload(dockerConfigPath+"/busybox-demo.yaml", remotePluginPath+"/docker/")
#install
cmdRemote = remoteBinPath + "/kubectl create -f "+remotePluginPath+"/docker/busybox-demo.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = "sleep 2s"
robj_admin.sudo(cmdRemote)
def installDockerPrivate(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
docker_data = self.config_object.get_node_by_attr("name", "docker")
dockerUsername = docker_data.find('username').text
dockerPassword = docker_data.find('password').text
dockerDomain = docker_data.find('domain').text
tmpPath = self.getLocalPath('tmpPath')
dockerConfigPath = self.getLocalPath('dockerConfigPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
#login docker registry
cmdRemote = "docker login --username=%s --password=%s %s" % (
dockerUsername, dockerPassword, dockerDomain)
robj_admin.sudo(cmdRemote)
#download docker login config file form master1 to local
robj_admin.download("/root/.docker/config.json", tmpPath + "/")
#view docker login config file content in local, and generate login credentials
# dockerSecret = $(cat ./config.json | base64 -w 0)
file_config = dockerConfigPath + "/registry-pull-secret.yaml"
file_tmp = tmpPath + "/registry-pull-secret.yaml"
with open(tmpPath + '/config.json') as file_object:
contents = file_object.read()
dockerSecret = base64.b64encode(
contents.rstrip().encode("utf-8")).decode("utf-8")
print("dockerSecret: " + dockerSecret)
sedRegex = "s/{dockerSecret}/%s/g" % dockerSecret.replace(
"/", r"\/")
cmdLocal = "cat %s | sed \"%s\" > %s" % (file_config, sedRegex,
file_tmp)
print(cmdLocal)
robj_admin.local(cmdLocal)
robj_admin.upload(file_tmp, remoteCfgPath + "/", True)
#create docker secret pod
cmdRemote = remoteBinPath + "/kubectl create -f " + remoteCfgPath + "/registry-pull-secret.yaml"
robj_admin.sudo(cmdRemote)
#get docker secret
cmdRemote = remoteBinPath + "/kubectl get secret"
robj_admin.sudo(cmdRemote)
def installCoredns(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
tmpPath = self.getLocalPath('tmpPath')
corednsConfigPath = self.getLocalPath('corednsConfigPath')
#upload
robj_admin.upload(corednsConfigPath + "/coredns-rbac.yaml",
remotePluginPath + "/coredns/", True)
robj_admin.upload(corednsConfigPath + "/coredns-sa.yaml",
remotePluginPath + "/coredns/", True)
robj_admin.upload(corednsConfigPath + "/coredns-configmap.yaml",
remotePluginPath + "/coredns/", True)
robj_admin.upload(corednsConfigPath + "/coredns-deployment.yaml",
remotePluginPath + "/coredns/", True)
ips_data = self.config_object.get_node_by_attr("name", "ips")
serviceClusterDnsIp = ips_data.find('serviceClusterDnsIp').text
sedRegex = "sed \"s/{serviceClusterDnsIp}/%s/g\"" % serviceClusterDnsIp
file_tmp = tmpPath + "/coredns-service.yaml"
file_config = corednsConfigPath + "/coredns-service.yaml"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_admin.local(cmd_local)
# 上传文件到远程主机
robj_admin.upload(file_tmp, remotePluginPath + "/coredns/", True)
#install
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-rbac.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-sa.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-configmap.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-deployment.yaml"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl create -f " + remotePluginPath + "/coredns/coredns-service.yaml"
robj_admin.sudo(cmdRemote)
def installCniNodeNormal(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj_node = Remote(node_env)
cniBinPath = self.getLocalPath('cniBinPath')
cniConfigPath = self.getLocalPath('cniConfigPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
tmpPath = self.getLocalPath('tmpPath')
remoteCniBinPath = self.getRemotePath('remoteCniBinPath')
remoteCniCfgPath = self.getRemotePath('remoteCniCfgPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
#stop relative service if the service is running
cmdRemote = "systemctl stop kube-kubelet"
robj_node.sudo(cmdRemote)
# upload cni files
robj_node.upload(cniBinPath + "/loopback", remoteCniBinPath + "/")
robj_node.upload(cniBinPath + "/portmap", remoteCniBinPath + "/")
robj_node.upload(cniConfigPath + "/99-loopback.conf",
remoteCniCfgPath + "/")
#upload kubelet service config file for cni
file_config = kbsConfigPath + "/kube-kubelet.conf"
file_tmp = tmpPath + "/kube-kubelet.conf"
plugin_name = "cni"
sedRegex = "sed \"s/^--hostname-override={nodeName}/--hostname-override=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/^--network-plugin={pluginName}/--network-plugin=%s/g\"" % plugin_name
sedRegex = sedRegex + " | sed \"s/^--cni-conf-dir={confDir}/--cni-conf-dir=%s/g\"" % remoteCniCfgPath.replace(
"/", "\/")
sedRegex = sedRegex + " | sed \"s/^--cni-bin-dir={binDir}/--cni-bin-dir=%s/g\"" % remoteCniBinPath.replace(
"/", "\/")
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
print(cmd_local)
robj_node.local(cmd_local)
# 上传文件到远程主机
robj_node.upload(file_tmp, remoteCfgPath + "/", True)
print("systemd daemon reload ...")
cmdRemote = "systemctl daemon-reload"
robj_node.sudo(cmdRemote)
print("restart kubelet service ...")
cmdRemote = "systemctl restart kube-kubelet.service"
robj_node.sudo(cmdRemote)
print("check kubelet service ...")
cmdRemote = "systemctl status kube-kubelet.service"
robj_node.sudo(cmdRemote)
def installIngress(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
remotePluginPath = self.getRemotePath('remotePluginPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
# tmpPath = self.getLocalPath('tmpPath')
ingressConfigPath = self.getLocalPath('ingressConfigPath')
#remove
cmdRemote = remoteBinPath + "/kubectl delete -f " + remotePluginPath + "/ingress/mandatory.yaml"
robj_admin.sudo(cmdRemote)
#upload yaml file
robj_admin.upload(ingressConfigPath + "/mandatory.yaml",
remotePluginPath + "/ingress/", True)
#install
cmdRemote = remoteBinPath + "/kubectl apply -f " + remotePluginPath + "/ingress/mandatory.yaml"
robj_admin.sudo(cmdRemote)
def installDemoIngress(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
remoteBinPath = self.getRemotePath('remoteBinPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
ingressConfigPath = self.getLocalPath('ingressConfigPath')
#upload ingress backend
robj_admin.upload(ingressConfigPath+"/nginx-ingress-backend.yaml", remotePluginPath+"/ingress/", True)
#install ingress backend
cmdRemote = remoteBinPath + "/kubectl apply -f "+remotePluginPath+"/ingress/nginx-ingress-backend.yaml"
robj_admin.sudo(cmdRemote)
#upload ingress rules
robj_admin.upload(ingressConfigPath+"/nginx-ingress-rules.yaml", remotePluginPath+"/ingress/", True)
#install ingress rules
cmdRemote = remoteBinPath + "/kubectl apply -f "+remotePluginPath+"/ingress/nginx-ingress-rules.yaml"
robj_admin.sudo(cmdRemote)
def installDashboard(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
dashboardConfigPath = self.getLocalPath('dashboardConfigPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
#label admin node for install dashboard
cmdRemote = remoteBinPath + "/kubectl label node " + admin_env.host + " role=admin"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl delete -f "
cmdRemote = cmdRemote + remotePluginPath + "/dashboard/kubernetes-dashboard.yaml"
robj_admin.sudo(cmdRemote)
robj_admin.upload(dashboardConfigPath + "/kubernetes-dashboard.yaml",
remotePluginPath + "/dashboard/")
cmdRemote = remoteBinPath + "/kubectl apply -f "
cmdRemote = cmdRemote + remotePluginPath + "/dashboard/kubernetes-dashboard.yaml"
robj_admin.sudo(cmdRemote)
def installKbsMasterNormal(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
tmpPath = self.getLocalPath('tmpPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
#stop relative service if the service is running
robj.sudo("systemctl stop kube-apiserver")
robj.sudo("systemctl stop kube-controller-manager")
robj.sudo("systemctl stop kube-scheduler")
#install some packages for proxy
cmdRemote = "yum install -y ipvsadm ipset conntrack"
robj.sudo(cmdRemote)
#download ca files from master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download etcd ca files from master1 to local
robj_master1.download(remoteSslPath + "/etcd.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd-key.pem", tmpPath + "/")
#download kubernetes ca files from master1 to local
robj_master1.download(remoteSslPath + "/kubernetes.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kubernetes.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kubernetes-key.pem",
tmpPath + "/")
#download metrics api ca files from master1 to local
robj_master1.download(remoteSslPath + "/front-proxy-ca.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.csr",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client-key.pem",
tmpPath + "/")
#download token csv file from master1 to local
robj_master1.download(remoteCfgPath + "/token.csv", tmpPath + "/")
#upload ca files to master node
robj.upload(tmpPath + "/ca.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/ca-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/kubernetes.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/kubernetes.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/kubernetes-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client-key.pem",
remoteSslPath + "/")
robj.upload(tmpPath + "/token.csv", remoteCfgPath + "/")
#upload binary files to master node
robj.upload((kbsBinPath + "/kube-apiserver"), (remoteBinPath + "/"))
robj.upload((kbsBinPath + "/kube-controller-manager"),
(remoteBinPath + "/"))
robj.upload((kbsBinPath + "/kube-scheduler"), (remoteBinPath + "/"))
#upload kubernetes api server config file
file_tmp = tmpPath + "/kube-apiserver.conf"
file_config = kbsConfigPath + "/kube-apiserver.conf"
sedRegex = "sed \"s/^--advertise-address={nodeIp}/--advertise-address=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp1}/%s/g\"" % self.getEnv(
"master1").host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp2}/%s/g\"" % self.getEnv(
"master2").host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp3}/%s/g\"" % self.getEnv(
"master3").host
sedRegex = sedRegex + " | sed \"s/{serviceClusterIpRange}/%s/g\"" % self.serviceClusterIpRange
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/", True)
# upload kubernetes controller manager config file
file_tmp = tmpPath + "/kube-controller-manager.conf"
file_config = kbsConfigPath + "/kube-controller-manager.conf"
sedRegex = "sed \"s/{serviceClusterIpRange}/%s/g\"" % self.serviceClusterIpRange
sedRegex = sedRegex + " | sed \"s/{podClusterIpRange}/%s/g\"" % self.podClusterIpRange
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/", True)
# upload kubernetes scheduler config file
robj.upload(kbsConfigPath + "/kube-scheduler.conf",
remoteCfgPath + "/", True)
#upload systemd service file to master node
robj.upload(kbsConfigPath + "/kube-apiserver.service",
remoteSystemdPath + "/", True)
robj.upload(kbsConfigPath + "/kube-controller-manager.service",
remoteSystemdPath + "/", True)
robj.upload(kbsConfigPath + "/kube-scheduler.service",
remoteSystemdPath + "/", True)
robj.sudo("systemctl daemon-reload")
robj.sudo("systemctl enable kube-apiserver")
robj.sudo("systemctl restart kube-apiserver")
robj.sudo("systemctl status kube-apiserver")
robj.sudo("systemctl enable kube-controller-manager")
robj.sudo("systemctl restart kube-controller-manager")
robj.sudo("systemctl status kube-controller-manager")
robj.sudo("systemctl enable kube-scheduler")
robj.sudo("systemctl restart kube-scheduler")
robj.sudo("systemctl status kube-scheduler")
def installEtcdNodeNormal(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
remoteEtcdDataPath = self.getRemotePath('remoteEtcdDataPath')
etcdBinPath = self.getLocalPath('etcdBinPath')
etcdConfigPath = self.getLocalPath('etcdConfigPath')
tmpPath = self.getLocalPath('tmpPath')
#upload ca files
robj.upload(tmpPath + "/ca.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/ca-key.pem", remoteSslPath + "/")
#upload etcd ca files
robj.upload(tmpPath + "/etcd.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd-key.pem", remoteSslPath + "/")
#upload binary file for etcd processes
if robj.checkpath((remoteBinPath + "/etcd")) == False:
robj.upload(etcdBinPath + "/etcd", remoteBinPath + "/")
if robj.checkpath((remoteBinPath + "/etcdctl")) == False:
robj.upload(etcdBinPath + "/etcdctl", remoteBinPath + "/")
#upload etcd config file
file_config = etcdConfigPath + "/etcd.conf"
file_tmp = tmpPath + "/etcd.conf"
sedRegex = "sed \"s/{etcdNodeName}/%s/g\"" % ("etcd-" + nodename)
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp}/%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeName1}/%s/g\"" % ("etcd-" +
"master1")
node1_env = self.getEnv("master1")
etcdNodeIp1 = node1_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp1}/%s/g\"" % etcdNodeIp1
sedRegex = sedRegex + " | sed \"s/{etcdNodeName2}/%s/g\"" % ("etcd-" +
"master2")
node2_env = self.getEnv("master2")
etcdNodeIp2 = node2_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp2}/%s/g\"" % etcdNodeIp2
sedRegex = sedRegex + " | sed \"s/{etcdNodeName3}/%s/g\"" % ("etcd-" +
"master3")
node3_env = self.getEnv("master3")
etcdNodeIp3 = node3_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp3}/%s/g\"" % etcdNodeIp3
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
robj.upload(file_tmp, remoteCfgPath + "/")
robj.upload(etcdConfigPath + "/etcd.service", remoteSystemdPath + "/")
if (remoteEtcdDataPath != "" and remoteEtcdDataPath != "/"
and robj.checkpath(remoteEtcdDataPath)):
robj.rmfile(remoteEtcdDataPath + "/*")
print("cache files of etcd is removed ...")
robj.sudo("ls -l " + remoteEtcdDataPath + "/")
else:
robj.mkdir(remoteEtcdDataPath)
#systemd for etcd process
robj.sudo("systemctl daemon-reload")
robj.sudo("systemctl enable etcd.service")
cmdRemote = "nohup systemctl restart etcd.service &> /dev/null &"
robj.sudo(cmdRemote)
def installNodeNormal(self, nodename):
# 初始化远程工具对象
robj = Remote(self.getEnv(nodename))
cfsslBinPath = self.getLocalPath('cfsslBinPath')
configPath = self.getLocalPath('configPath')
remoteSysctlPath = self.getRemotePath('remoteSysctlPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
# remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
# tmpPath = self.getLocalPath('tmpPath')
# dockerConfigPath = self.getLocalPath('dockerConfigPath')
#install some packages for proxy
# cmdRemote = "yum install -y ipvsadm ipset conntrack"
# cmdRemote = "yum install -y conntrack"
# robj.sudo(cmdRemote)
#upload cfssl file
robj.upload(cfsslBinPath + "/cfssl", remoteLocalBinPath + "/")
robj.upload(cfsslBinPath + "/cfssljson", remoteLocalBinPath + "/")
robj.upload(cfsslBinPath + "/cfssl-certinfo", remoteLocalBinPath + "/")
#cfssl tool
if robj.checkpath(remoteLocalBinPath + "/cfssl") == True:
print("ready to chmod cfssl ...")
cmdRemote = "chmod +x /usr/local/bin/cfssl*"
robj.sudo(cmdRemote)
#close selinux
print("ready to close selinux ...")
robj.sudo("setenforce 0")
robj.sudo(
"sed -i \"s/^SELINUX=enforcing/SELINUX=disabled/g\" /etc/sysconfig/selinux"
)
robj.sudo(
"sed -i \"s/^SELINUX=enforcing/SELINUX=disabled/g\" /etc/selinux/config"
)
robj.sudo(
"sed -i \"s/^SELINUX=permissive/SELINUX=disabled/g\" /etc/sysconfig/selinux"
)
robj.sudo(
"sed -i \"s/^SELINUX=permissive/SELINUX=disabled/g\" /etc/selinux/config"
)
robj.sudo("getenforce")
#close swap
print("ready to close swap ...")
robj.sudo("swapoff -a")
#comment swap line
robj.sudo("sed -i 's/.*swap.*/#&/' /etc/fstab")
robj.sudo("cat /etc/fstab")
#config kernel
print("ready to config kernel ...")
robj.upload(configPath + "/sysctl/k8s.conf", remoteSysctlPath + "/")
robj.sudo("modprobe br_netfilter")
#reload system config
print("reload system config ...")
robj.sudo("sysctl -p " + remoteSysctlPath + "/k8s.conf")
#close firewall
print("ready to stop firewall ...")
robj.sudo("systemctl stop firewalld")
robj.sudo("systemctl disable firewalld")
robj.sudo("systemctl status firewalld")
#remove old version of docker
robj.sudo("systemctl stop docker")
robj.sudo(
"yum remove -y docker docker-client docker-client-latest docker-common"
)
robj.sudo(
"yum remove -y docker-latest docker-latest-logrotate docker-logrotate docker-engine"
)
robj.sudo("systemctl status docker")
# yum priorities plugin
print("ready to install yum priorities plugin ...")
robj.sudo("yum install -y yum-plugin-priorities")
#epel-release
print("ready to install epel repository ...")
robj.sudo("yum install -y epel-release")
robj.sudo("yum repolist")
#ntpdate
if self.mode == "dev":
print("ready to install ntpdate ...")
robj.sudo("yum -y install ntpdate")
robj.sudo("systemctl enable ntpdate")
robj.sudo("systemctl restart ntpdate")
#set prioritiy for epel
if robj.checkpath("/etc/yum.repos.d/epel.repo") == True:
cmdRemote = "grep -c \"priority=\" /etc/yum.repos.d/epel.repo"
robj.sudo(cmdRemote)
priorityExists = robj.getResult().stdout.rstrip()
if priorityExists == '0':
# insert priority field under the 'enabled=' line
cmdRemote = "sed -i '/enabled=/a\priority=1' /etc/yum.repos.d/epel.repo"
robj.sudo(cmdRemote)
# update yum
cmdRemote = "yum makecache"
robj.sudo(cmdRemote)
# cmdRemote = "yum update -y"
# robj.sudo(cmdRemote)
cmdRemote = "yum install -y deltarpm"
robj.sudo(cmdRemote)
# cmdRemote = "yum update -y"
# robj.sudo(cmdRemote)
#wget tool
print("ready to install wget ...")
cmdRemote = "yum install -y wget"
robj.sudo(cmdRemote)
#net tool
print("ready to install net-tools ...")
cmdRemote = "yum install -y net-tools"
robj.sudo(cmdRemote)
#install docker component
if robj.checkpath("/usr/bin/docker") == False:
#Set up repository
print("ready to install docker-ce ...")
#install required packages. yum-utils provides the yum-config-manager utility,
# and device-mapper-persistent-data and lvm2 are required by the devicemapper storage driver
cmdRemote = "yum install -y yum-utils device-mapper-persistent-data lvm2"
robj.sudo(cmdRemote)
#set up the stable repository
# sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
cmdRemote = "yum-config-manager --add-repo"
cmdRemote = cmdRemote + " http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo"
robj.sudo(cmdRemote)
# update yum
cmdRemote = "yum makecache"
robj.sudo(cmdRemote)
# cmdRemote = "yum update -y"
# robj.sudo(cmdRemote)
#List and sort the versions available in your repo. This example sorts results by version number, highest to lowest, and is truncated
print("list docker versions ...")
cmdRemote = "yum list docker-ce --showduplicates | sort -r"
robj.sudo(cmdRemote)
robj.sudo("sleep 5")
# if not confirm("Ready to install docker-ce-18.06.3.ce, Continue[Y/N]?"):
# return True
#Install a specific version by its fully qualified package name
print("ready to install docker-ce-18.06.3.ce ...")
cmdRemote = "yum install -y docker-ce-18.06.3.ce docker-ce-cli-18.06.3.ce containerd.io"
robj.sudo(cmdRemote)
#set docker to start when the system is booted
cmdRemote = "systemctl enable docker"
robj.sudo(cmdRemote)
#set subnet for docker
# upload docker service file
# sedRegex = "sed \"s/{dockerSubnetIpRange}/%s/g\"" % self.dockerSubnetIpRange
# file_tmp = tmpPath + "/docker.service"
# file_config = dockerConfigPath + "/docker.service"
# cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
# robj.local(cmd_local)
# 上传文件到远程主机
# robj.upload(file_tmp, remoteSystemdPath+"/", True)
#reload daemon
cmdRemote = "systemctl daemon-reload"
robj.sudo(cmdRemote)
#start docker
cmdRemote = "systemctl restart docker"
robj.sudo(cmdRemote)
# check docker status
print("check docker-ce status ...")
cmdRemote = "systemctl status docker"
robj.sudo(cmdRemote)
# check subnet of docker
print("check subnet of docker0 ...")
cmdRemote = "ip addr show docker0"
robj.sudo(cmdRemote)
#install socat
cmdRemote = "yum install -y socat"
robj.sudo(cmdRemote)
#prepare deployment directory
cmdRemote = "mkdir -p /opt/kubernetes/{bin,cfg,ssl,log}"
robj.sudo(cmdRemote)
#set environment variable
robj.upload(configPath + "/sysctl/k8s.sh", "/etc/profile.d/", True)
# cmdRemote = "echo 'export PATH=/opt/kubernetes/bin:$PATH' > /etc/profile.d/k8s.sh"
# robj.sudo(cmdRemote)
robj.sudo("chmod 755 /etc/profile.d/k8s.sh")
cmdRemote = "source /etc/profile.d/k8s.sh"
robj.run(cmdRemote)
def installMetricsServer(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
metricsConfigPath = self.getLocalPath('metricsConfigPath')
# kbsPluginPath = self.getLocalPath('kbsPluginPath')
# tmpPath = self.getLocalPath('tmpPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remotePluginPath = self.getRemotePath('remotePluginPath')
# metricsConfigPath = kbsPluginPath + "/metrics-server"
robj_admin.upload(
metricsConfigPath + "/aggregated-metrics-reader.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(metricsConfigPath + "/auth-delegator.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(metricsConfigPath + "/auth-reader.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(metricsConfigPath + "/metrics-apiservice.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(
metricsConfigPath + "/metrics-server-deployment.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(metricsConfigPath + "/metrics-server-service.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(metricsConfigPath + "/resource-reader.yaml",
remotePluginPath + "/metrics/", True)
robj_admin.upload(metricsConfigPath + "/metrics-server-ingress.yaml",
remotePluginPath + "/metrics/", True)
# robj_admin.upload(metricsConfigPath+"/auth-delegator.yaml", remotePluginPath+"/metrics/", True)
# robj_admin.upload(metricsConfigPath+"/auth-reader.yaml", remotePluginPath+"/metrics/", True)
# robj_admin.upload(metricsConfigPath+"/metrics-apiservice.yaml", remotePluginPath+"/metrics/", True)
# robj_admin.upload(metricsConfigPath+"/metrics-server-service.yaml", remotePluginPath+"/metrics/", True)
# robj_admin.upload(metricsConfigPath+"/resource-reader.yaml", remotePluginPath+"/metrics/", True)
# file_config = metricsConfigPath + "/metrics-server-deployment.yaml"
# file_tmp = tmpPath + "/metrics-server-deployment.yaml"
# s1 = "k8s.gcr.io/metrics-server-amd64:v0.3.1".replace("/", "\/")
# r1 = "registry.cn-beijing.aliyuncs.com/kube-systems/metrics-server:0.3.1".replace("/", "\/")
# s2 = "k8s.gcr.io/addon-resizer:1.8.4".replace("/", "\/")
# r2 = "registry.cn-beijing.aliyuncs.com/kube-systems/addon-resizer:1.8.4".replace("/", "\/")
# sedRegex = "sed \"s/"+s1+"/"+r1+"/g\""
# sedRegex = sedRegex + " | sed \"s/"+s2+"/"+r2+"/g\""
# cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
# robj_admin.local(cmd_local)
# # 上传文件到远程主机
# robj_admin.upload(file_tmp, remotePluginPath+"/metrics/", True)
cmdRemote = remoteBinPath + "/kubectl create -f "
cmdRemote = cmdRemote + remotePluginPath + "/metrics/"
robj_admin.sudo(cmdRemote)
def installKbsKubelet(self, nodename):
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
print("node ip: " + node_env.host)
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
print("master ip: " + node_env1.host)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
tmpPath = self.getLocalPath('tmpPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
#download ca files from master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download etcd ca files from master1 to local
robj_master1.download(remoteSslPath + "/etcd.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd-key.pem", tmpPath + "/")
robj_master1.download(remoteCfgPath + "/token.csv", tmpPath + "/")
#download kubeconfig files from admin to local
robj_admin.download(remoteCfgPath + "/bootstrap.kubeconfig",
tmpPath + "/")
#stop relative service if the service is running
cmdRemote = "systemctl stop kube-kubelet"
robj.sudo(cmdRemote)
#upload binary file for kube processes
if robj.checkpath(remoteBinPath + "/kubelet") == False:
robj.upload(kbsBinPath + "/kubelet", remoteBinPath + "/")
#upload ca files to node1
robj.upload(tmpPath + "/ca.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/ca-key.pem", remoteSslPath + "/")
#upload etcd ca files to node1
robj.upload(tmpPath + "/etcd.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/etcd-key.pem", remoteSslPath + "/")
#upload config files to node1
robj.upload(tmpPath + "/bootstrap.kubeconfig", remoteCfgPath + "/")
robj.upload(tmpPath + "/token.csv", remoteCfgPath + "/")
sedRegex = "sed \"s/{serviceClusterDnsIp}/%s/g\"" % self.serviceClusterDnsIp
file_tmp = tmpPath + "/kubelet.config"
file_config = kbsConfigPath + "/kubelet.config"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/")
#upload systemd service files to node
robj.upload(kbsConfigPath + "/kube-kubelet.service",
remoteSystemdPath + "/")
print("node ip: " + node_env.host)
#upload kubelet service config file
file_config = kbsConfigPath + "/kube-kubelet.conf"
file_tmp = tmpPath + "/kube-kubelet.conf"
sedRegex = "sed \"s/^--hostname-override={nodeName}/--hostname-override=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/^--network-plugin={pluginName}//g\""
sedRegex = sedRegex + " | sed \"s/^--cni-conf-dir={confDir}//g\""
sedRegex = sedRegex + " | sed \"s/^--cni-bin-dir={binDir}//g\""
sedRegex = sedRegex + " | sed \"s/{serviceClusterDnsIp}/%s/g\"" % self.serviceClusterDnsIp
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath, True)
print("systemd daemon reload ...")
cmdRemote = "systemctl daemon-reload"
robj.sudo(cmdRemote)
print("enable kubelet service ...")
cmdRemote = "systemctl enable kube-kubelet.service"
robj.sudo(cmdRemote)
print("restart kubelet service ...")
cmdRemote = "systemctl restart kube-kubelet.service"
robj.sudo(cmdRemote)
print("check kubelet service ...")
cmdRemote = "systemctl status kube-kubelet.service"
robj.sudo(cmdRemote)
print("sleep 5 seconds ...")
cmdRemote = "sleep 5"
robj.sudo(cmdRemote)
print("restart kube apiserver ...")
cmdRemote = "systemctl restart kube-apiserver"
robj_master1.sudo(cmdRemote)
print("show csr info: ")
cmdRemote = remoteBinPath + "/kubectl get csr"
robj_admin.sudo(cmdRemote)
print("pass tls request: ")
cmdRemote = remoteBinPath + "/kubectl get csr | grep 'Pending'"
cmdRemote = cmdRemote + " | awk 'NR>0 {print $1}'"
cmdRemote = cmdRemote + " | xargs " + remoteBinPath + "/kubectl certificate approve"
robj_admin.sudo(cmdRemote)
def installKbsProxy(self, nodename):
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
node_env = self.getEnv(nodename)
# 初始化远程工具对象
robj = Remote(node_env)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSystemdPath = self.getRemotePath('remoteSystemdPath')
tmpPath = self.getLocalPath('tmpPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
#download proxy cert files from master1
robj_master1.download(remoteSslPath + "/kube-proxy.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kube-proxy.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kube-proxy-key.pem",
tmpPath + "/")
#download proxy kubeconfig files from admin node
robj_admin.download(remoteCfgPath + "/kube-proxy.kubeconfig",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-ca.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-ca-key.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.csr",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/front-proxy-client-key.pem",
tmpPath + "/")
robj.upload(tmpPath + "/front-proxy-ca.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-ca-key.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.csr", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/front-proxy-client-key.pem",
remoteSslPath + "/")
#stop relative service if the service is running
cmdRemote = "systemctl stop kube-proxy"
robj.sudo(cmdRemote)
#install some packages for proxy
cmdRemote = "yum install -y ipvsadm ipset conntrack"
robj.sudo(cmdRemote)
#upload binary file for kube processes
if robj.checkpath(remoteBinPath + "/kube-proxy") == False:
robj.upload(kbsBinPath + "/kube-proxy", remoteBinPath + "/")
#upload proxy cert files to node1
robj.upload(tmpPath + "/kube-proxy.pem", remoteSslPath + "/")
robj.upload(tmpPath + "/kube-proxy-key.pem", remoteSslPath + "/")
#upload proxy kubeconfig files to node1
robj.upload(tmpPath + "/kube-proxy.kubeconfig", remoteCfgPath + "/")
#upload systemd service files to node1
robj.upload(kbsConfigPath + "/kube-proxy.service",
remoteSystemdPath + "/")
#upload proxy service config file
sedRegex = "sed \"s/^--hostname-override={nodeName}/--hostname-override=%s/g\"" % node_env.host
sedRegex = sedRegex + " | sed \"s/{podClusterIpRange}/%s/g\"" % self.podClusterIpRange
file_tmp = tmpPath + "/kube-proxy.conf"
file_config = kbsConfigPath + "/kube-proxy.conf"
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj.local(cmd_local)
# 上传文件到远程主机
robj.upload(file_tmp, remoteCfgPath + "/", True)
print("systemd daemon reload ...")
cmdRemote = "systemctl daemon-reload"
robj.sudo(cmdRemote)
print("enable proxy service ...")
cmdRemote = "systemctl enable kube-proxy.service"
robj.sudo(cmdRemote)
print("restart proxy service ...")
cmdRemote = "systemctl restart kube-proxy.service"
robj.sudo(cmdRemote)
print("check proxy service ...")
cmdRemote = "systemctl status kube-proxy.service"
robj.sudo(cmdRemote)
def installKbsMasterBefore(self):
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
tmpPath = self.getLocalPath('tmpPath')
etcdConfigPath = self.getLocalPath('etcdConfigPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
#download ca files form master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
# robj_master1.download(remoteSslPath+"/ca.pem", tmpPath+"/")
# robj_master1.download(remoteSslPath+"/ca-key.pem", tmpPath+"/")
#upload ca cert files
# robj_master1.upload(tmpPath+"/ca.pem", remoteSslPath+"/")
# robj_master1.upload(tmpPath+"/ca-key.pem", remoteSslPath+"/")
#upload files
robj_master1.upload(etcdConfigPath + "/ca-config.json",
remoteSslPath + "/")
# upload kubernetes csr config file
file_tmp = tmpPath + "/kubernetes-csr.json"
file_config = kbsConfigPath + "/kubernetes-csr.json"
sedRegex = "sed \"s/{kbsMasterIp1}/%s/g\"" % self.getEnv(
"master1").host
sedRegex = sedRegex + " | sed \"s/{kbsMasterIp2}/%s/g\"" % self.getEnv(
"master2").host
sedRegex = sedRegex + " | sed \"s/{kbsMasterIp3}/%s/g\"" % self.getEnv(
"master3").host
sedRegex = sedRegex + " | sed \"s/{kbsMasterProxyIp}/%s/g\"" % self.proxy_host
sedRegex = sedRegex + " | sed \"s/{serviceClusterGatewayIp}/%s/g\"" % self.serviceClusterGatewayIp
sedRegex = sedRegex + " | sed \"s/{workerNodeIp}/%s/g\"" % self.getEnv(
"node1").host
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_master1.local(cmd_local)
# 上传文件到远程主机
robj_master1.upload(file_tmp, remoteSslPath + "/")
# robj_master1.upload(kbsConfigPath+"/admin-csr.json", remoteSslPath+"/")
#create token.csv file for bootstrap authority in master1
print("ready to create tocken.csv file in master1 ...")
cmdRemote = "/bin/sh -c \'BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d \" \")"
cmdRemote = cmdRemote + "; BOOTSTRAP_CONTENT=\"${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,\\\"system:kubelet-bootstrap\\\"\""
cmdRemote = cmdRemote + "; echo ${BOOTSTRAP_CONTENT} > " + remoteCfgPath + "/token.csv\'"
# print(cmdRemote)
robj_master1.sudo(cmdRemote)
robj_master1.sudo("cat " + remoteCfgPath + "/token.csv")
#create kubernetes cert files in master1
print("ready to create kubernetes cert file in master1 ...")
cmdRemote = remoteLocalBinPath + "/cfssl gencert"
cmdRemote = cmdRemote + " -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/kubernetes-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/kubernetes"
robj_master1.sudo(cmdRemote)
#create metrics api aggregator cert files in master1
robj_master1.upload(kbsConfigPath + "/front-proxy-ca-csr.json",
remoteSslPath + "/")
robj_master1.upload(kbsConfigPath + "/front-proxy-client-csr.json",
remoteSslPath + "/", True)
# generate ca root cert
cmdRemote = remoteLocalBinPath + "/cfssl gencert -initca " + remoteSslPath + "/front-proxy-ca-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/front-proxy-ca"
robj_master1.sudo(cmdRemote)
# generate client cert for each node
cmdRemote = remoteLocalBinPath + "/cfssl gencert -ca=" + remoteSslPath + "/front-proxy-ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/front-proxy-ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/front-proxy-client-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/front-proxy-client"
robj_master1.sudo(cmdRemote)
def installKbsNodeBefore(self):
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteCfgPath = self.getRemotePath('remoteCfgPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
tmpPath = self.getLocalPath('tmpPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
#download token csv file from master1 to local
robj_master1.download(remoteCfgPath + "/token.csv", tmpPath + "/")
#####for kubelet#####
#delete old role binding in master1
cmdRemote = remoteBinPath + "/kubectl delete clusterrolebinding/kubelet-bootstrap"
robj_admin.sudo(cmdRemote)
#create role binding in master1
cmdRemote = remoteBinPath + "/kubectl create clusterrolebinding kubelet-bootstrap"
cmdRemote = cmdRemote + " --clusterrole=system:node-bootstrapper"
cmdRemote = cmdRemote + " --user=kubelet-bootstrap"
robj_admin.sudo(cmdRemote)
#set cluster parameters and create kubelet bootstrapping kubeconfig file
cmdRemote = remoteBinPath + "/kubectl config set-cluster kubernetes"
cmdRemote = cmdRemote + " --certificate-authority=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --server=https://" + self.proxy_host + ":" + self.proxy_port
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
robj_admin.local("sleep 3")
f = open(tmpPath + '/token.csv', 'r')
csv_file = csv.reader(f)
csv_content = next(csv_file)
csv_token = csv_content[0]
f.close()
#create authority parameters for client
cmdRemote = remoteBinPath + "/kubectl config set-credentials kubelet-bootstrap"
cmdRemote = cmdRemote + " --token=" + csv_token
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
#set context parameters
cmdRemote = remoteBinPath + "/kubectl config set-context kubernetes"
cmdRemote = cmdRemote + " --cluster=kubernetes"
cmdRemote = cmdRemote + " --user=kubelet-bootstrap"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
#set default context
cmdRemote = remoteBinPath + "/kubectl config use-context kubernetes"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/bootstrap.kubeconfig"
robj_admin.sudo(cmdRemote)
#####for proxy#####
#upload proxy csr file to master1
robj_master1.upload(kbsConfigPath + "/kube-proxy-csr.json",
remoteSslPath + "/")
#delete old proxy cert files in master1
robj_master1.rmfile(remoteSslPath + "/kube-proxy.csr")
robj_master1.rmfile(remoteSslPath + "/kube-proxy.pem")
robj_master1.rmfile(remoteSslPath + "/kube-proxy-key.pem")
#create proxy cert files in master1
cmdRemote = "/usr/local/bin/cfssl gencert"
cmdRemote = cmdRemote + " -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/kube-proxy-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/kube-proxy"
robj_master1.sudo(cmdRemote)
#download cert files from master1 to local
robj_master1.download(remoteSslPath + "/kube-proxy.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/kube-proxy-key.pem",
tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
#upload cert files from local to admin node
robj_admin.upload(tmpPath + "/kube-proxy.pem", remoteSslPath + "/")
robj_admin.upload(tmpPath + "/kube-proxy-key.pem", remoteSslPath + "/")
robj_admin.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
#create kube-proxy kubeconfig file in admin node
cmdRemote = remoteBinPath + "/kubectl config set-cluster kubernetes"
cmdRemote = cmdRemote + " --certificate-authority=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --server=https://" + self.proxy_host + ":" + self.proxy_port
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl config set-credentials kube-proxy"
cmdRemote = cmdRemote + " --client-certificate=" + remoteSslPath + "/kube-proxy.pem"
cmdRemote = cmdRemote + " --client-key=" + remoteSslPath + "/kube-proxy-key.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl config set-context kubernetes"
cmdRemote = cmdRemote + " --cluster=kubernetes"
cmdRemote = cmdRemote + " --user=kube-proxy"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl config use-context kubernetes"
cmdRemote = cmdRemote + " --kubeconfig=" + remoteCfgPath + "/kube-proxy.kubeconfig"
robj_admin.sudo(cmdRemote)
def installEtcdNodePrepare(self):
#stop relative service if the service is running
print("stop relative service if the service is running ...")
node_env1 = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(node_env1)
print("stop etcd service in master1")
robj_master1.sudo("systemctl stop etcd")
node_env2 = self.getEnv("master2")
# 初始化远程工具对象
robj_master2 = Remote(node_env2)
print("stop etcd service in master2")
robj_master2.sudo("systemctl stop etcd")
node_env3 = self.getEnv("master3")
# 初始化远程工具对象
robj_master3 = Remote(node_env3)
print("stop etcd service in master3")
robj_master3.sudo("systemctl stop etcd")
remoteSslPath = self.getRemotePath('remoteSslPath')
remoteLocalBinPath = self.getRemotePath('remoteLocalBinPath')
etcdConfigPath = self.getLocalPath('etcdConfigPath')
tmpPath = self.getLocalPath('tmpPath')
#upload ca cert config file
robj_master1.upload(etcdConfigPath + "/ca-csr.json",
remoteSslPath + "/")
robj_master1.upload(etcdConfigPath + "/ca-config.json",
remoteSslPath + "/")
#upload etcd csr file
file_config = etcdConfigPath + "/etcd-csr.json"
file_tmp = tmpPath + "/etcd-csr.json"
node1_env = self.getEnv("master1")
etcdNodeIp1 = node1_env.host
sedRegex = "sed \"s/{etcdNodeIp1}/%s/g\"" % etcdNodeIp1
node2_env = self.getEnv("master2")
etcdNodeIp2 = node2_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp2}/%s/g\"" % etcdNodeIp2
node3_env = self.getEnv("master3")
etcdNodeIp3 = node3_env.host
sedRegex = sedRegex + " | sed \"s/{etcdNodeIp3}/%s/g\"" % etcdNodeIp3
node4_env = self.getEnv("node1")
workerNodeIp = node4_env.host
sedRegex = sedRegex + " | sed \"s/{workerNodeIp}/%s/g\"" % workerNodeIp
cmd_local = "cat %s | %s > %s" % (file_config, sedRegex, file_tmp)
robj_master1.local(cmd_local)
robj_master1.upload(file_tmp, remoteSslPath + "/")
# generate ca root cert
cmdRemote = remoteLocalBinPath + "/cfssl gencert -initca " + remoteSslPath + "/ca-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/ca"
robj_master1.sudo(cmdRemote)
#generate etcd cert for each node
cmdRemote = remoteLocalBinPath + "/cfssl gencert -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/etcd-csr.json"
cmdRemote = cmdRemote + " | " + remoteLocalBinPath + "/cfssljson -bare " + remoteSslPath + "/etcd"
robj_master1.sudo(cmdRemote)
#download ca files form master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download etcd ca files form master1 to local
robj_master1.download(remoteSslPath + "/etcd.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/etcd-key.pem", tmpPath + "/")
def installKubectl(self):
admin_env = self.getEnv("admin")
# 初始化远程工具对象
robj_admin = Remote(admin_env)
master1_env = self.getEnv("master1")
# 初始化远程工具对象
robj_master1 = Remote(master1_env)
remoteBinPath = self.getRemotePath('remoteBinPath')
remoteSslPath = self.getRemotePath('remoteSslPath')
tmpPath = self.getLocalPath('tmpPath')
kbsBinPath = self.getLocalPath('kbsBinPath')
kbsConfigPath = self.getLocalPath('kbsConfigPath')
robj_master1.upload(kbsConfigPath + "/admin-csr.json",
remoteSslPath + "/")
#create kubectl cert files in master1
cmdRemote = "/usr/local/bin/cfssl gencert"
cmdRemote = cmdRemote + " -ca=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " -ca-key=" + remoteSslPath + "/ca-key.pem"
cmdRemote = cmdRemote + " -config=" + remoteSslPath + "/ca-config.json"
cmdRemote = cmdRemote + " -profile=kubernetes " + remoteSslPath + "/admin-csr.json"
cmdRemote = cmdRemote + " | /usr/local/bin/cfssljson -bare " + remoteSslPath + "/admin"
robj_master1.sudo(cmdRemote)
#download binary files to local
# robj_master1.download(remoteBinPath+"/kubectl", tmpPath+"/")
#download ca files from master1 to local
robj_master1.download(remoteSslPath + "/ca.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/ca-key.pem", tmpPath + "/")
#download kubectl ca files from master1 to local
robj_master1.download(remoteSslPath + "/admin.csr", tmpPath + "/")
robj_master1.download(remoteSslPath + "/admin.pem", tmpPath + "/")
robj_master1.download(remoteSslPath + "/admin-key.pem", tmpPath + "/")
# upload kubectl file
if robj_admin.checkpath(remoteBinPath + "/kubectl") == False:
robj_admin.upload(kbsBinPath + "/kubectl", remoteBinPath + "/")
#upload cert files to master node
robj_admin.upload(tmpPath + "/ca.pem", remoteSslPath + "/")
robj_admin.upload(tmpPath + "/admin.pem", remoteSslPath + "/")
robj_admin.upload(tmpPath + "/admin-key.pem", remoteSslPath + "/")
#set cluster parameters
cmdRemote = remoteBinPath + "/kubectl config set-cluster kubernetes"
cmdRemote = cmdRemote + " --certificate-authority=" + remoteSslPath + "/ca.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --server=https://" + self.proxy_host + ":" + self.proxy_port
robj_admin.sudo(cmdRemote)
#set client authority parameters
cmdRemote = remoteBinPath + "/kubectl config set-credentials admin"
cmdRemote = cmdRemote + " --client-certificate=" + remoteSslPath + "/admin.pem"
cmdRemote = cmdRemote + " --embed-certs=true"
cmdRemote = cmdRemote + " --client-key=" + remoteSslPath + "/admin-key.pem"
robj_admin.sudo(cmdRemote)
#set context parameters
cmdRemote = remoteBinPath + "/kubectl config set-context kubernetes"
cmdRemote = cmdRemote + " --cluster=kubernetes"
cmdRemote = cmdRemote + " --user=admin"
robj_admin.sudo(cmdRemote)
#set default context
cmdRemote = remoteBinPath + "/kubectl config use-context kubernetes"
robj_admin.sudo(cmdRemote)
cmdRemote = remoteBinPath + "/kubectl get nodes -o wide"
robj_admin.sudo(cmdRemote)