def validata_node_os(config_path):
layout_file = os.path.join(config_path, "layout.yaml")
hosts = yaml.load(open(layout_file))["machine-list"]
print hosts
host_not_match = []
regex = r"Description:\s+Ubuntu\s+16.04(.\d)?\s+LTS"
for host in hosts:
logger.info("Check os for %s ...", host["hostip"])
result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
host, "sudo lsb_release -d")
logger.info(result_stdout)
# should match Ubuntu 16.04.5 LTS
if re.match(regex, result_stdout) is None:
host_not_match.append(host["hostip"])
if len(host_not_match) > 0:
logger.error(
"[Error - unsupported OS] - OpenPAI only supports ubuntu 16.04, the following node(s) are not ubuntu 16.04: %s",
host_not_match)
exit(-1)
def generate_configuration(quick_start_config_file, configuration_directory, force):
"""Automatically generate the following configuration files from a quick-start file:
* Machine-level configurations: cluster-configuration.yaml
* Kubernetes-level configurations I: kubernetes-configuration.yaml
* Kubernetes-level configurations II: k8s-role-definition.yaml
* Service-level configurations: service-configuration.yaml
"""
quick_start_config_raw = file_handler.load_yaml_config(quick_start_config_file)
quick_start_config = {}
#
# Prepare config of ssh info.
quick_start_config["ssh-username"] = quick_start_config_raw["ssh-username"]
quick_start_config["ssh-password"] = quick_start_config_raw["ssh-password"]
quick_start_config["ssh-port"] = \
22 if "ssh-port" not in quick_start_config_raw \
else quick_start_config_raw["ssh-port"]
#
# Prepare config of cluster IP range.
quick_start_config["service-cluster-ip-range"] = \
"10.254.0.0/16" if "service-cluster-ip-range" not in quick_start_config_raw \
else quick_start_config_raw["service-cluster-ip-range"]
#
# Prepare config of machine list.
quick_start_config["machines"] = []
for m in quick_start_config_raw["machines"]:
quick_start_config["machines"].append({"hostname": None, "ip": m})
#
# Auto-complete missing configuration items: Part 1 -- DNS.
if "dns" in quick_start_config_raw:
quick_start_config["dns"] = quick_start_config_raw["dns"]
else:
m0 = quick_start_config["machines"][0]
result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
{
"hostip": m0["ip"],
"username": quick_start_config["ssh-username"],
"password": quick_start_config["ssh-password"],
"sshport": quick_start_config["ssh-port"]
},
"cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2")
quick_start_config["dns"] = result_stdout.strip()
#
# Auto-complete missing configuration items: Part 2 -- hostnames.
for m in quick_start_config["machines"]:
result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
{
"hostip": m["ip"],
"username": quick_start_config["ssh-username"],
"password": quick_start_config["ssh-password"],
"sshport": quick_start_config["ssh-port"]
},
"hostname")
m["hostname"] = result_stdout.strip()
#
# Generate configuration files.
target_file_names = [
"cluster-configuration.yaml",
"kubernetes-configuration.yaml",
"k8s-role-definition.yaml",
"services-configuration.yaml"
]
for x in target_file_names:
target_file_path = os.path.join(configuration_directory, x)
if file_handler.file_exist_or_not(target_file_path) and force is False:
print("File %s exists. Skip." % (target_file_path))
pass
else:
file_handler.create_folder_if_not_exist(configuration_directory)
file_handler.write_generated_file(
target_file_path,
template_handler.generate_from_template_dict(
file_handler.read_template("./quick-start/%s.template" % (x)),
{ "env": quick_start_config }))
def generate_configuration(quick_start_config_file, configuration_directory, force):
"""Automatically generate the following configuration files from a quick-start file:
* Machine-level configurations: laylout.yaml
* Kubernetes-level configurations I: kubernetes-configuration.yaml
* Kubernetes-level configurations II: k8s-role-definition.yaml
* Service-level configurations: service-configuration.yaml
"""
quick_start_config_raw = file_handler.load_yaml_config(quick_start_config_file)
#
# Prepare machine list
machine_list = []
for m in quick_start_config_raw["machines"]:
machine = {"hostip": m, "machine-type": "GENERIC"}
# TODO on premise, using ip as "nodename"
machine["nodename"] = m
machine["docker-data"] = "/var/lib/docker"
machine["username"] = quick_start_config_raw["ssh-username"]
if "ssh-password" in quick_start_config_raw:
machine["password"] = quick_start_config_raw["ssh-password"]
else:
machine["ssh-keyfile-path"] = quick_start_config_raw["ssh-keyfile-path"]
machine["ssh-secret-name"] = quick_start_config_raw["ssh-secret-name"]
machine["ssh-port"] = 22 if "ssh-port" not in quick_start_config_raw else quick_start_config_raw["ssh-port"]
machine_list.append(machine)
# workers
worker_noders = machine_list[1:] if len(machine_list) > 1 else machine_list
for machine in worker_noders:
# k8s attributes
machine["k8s-role"] = "worker"
# PAI attributes
machine["pai-worker"] = "true"
# master
master_node = machine_list[0]
# k8s attributes
master_node["k8s-role"] = "master"
master_node["etcdid"] = "etcdid1"
master_node["dashboard"] = "true"
# PAI attributes
master_node["pai-master"] = "true"
master_node["zkid"] = "1"
#
# Prepare config of cluster IP range.
service_cluster_ip_range = \
"10.254.0.0/16" if "service-cluster-ip-range" not in quick_start_config_raw \
else quick_start_config_raw["service-cluster-ip-range"]
#
# Auto-complete missing configuration items: Part 1 -- DNS.
if "dns" in quick_start_config_raw:
dns = quick_start_config_raw["dns"]
else:
result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
master_node,
"cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2 | head -n 1")
dns = result_stdout.strip()
#
# Auto-complete missing configuration items: Part 2 -- hostnames.
for host_config in machine_list:
result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
host_config,
"hostname")
host_config["hostname"] = result_stdout.strip()
#
# kubernetes info
api_servers_url = "http://{0}:{1}".format(master_node["hostip"], 8080)
# TODO we some k8s template still using the 'dashboard_host'
dashboard_host = master_node["hostip"]
dashboard_url = "http://{0}:{1}".format(master_node["hostip"], 9090)
#
# Generate configuration files.
target_file_names = [
"layout.yaml",
"kubernetes-configuration.yaml",
"k8s-role-definition.yaml",
"services-configuration.yaml"
]
for x in target_file_names:
target_file_path = os.path.join(configuration_directory, x)
if file_handler.file_exist_or_not(target_file_path) and force is False:
logger.warning("File %s exists. Skip." % (target_file_path))
pass
else:
file_handler.create_folder_if_not_exist(configuration_directory)
file_handler.write_generated_file(
target_file_path,
template_handler.generate_from_template_dict(
file_handler.read_template("./deployment/quick-start/%s.template" % (x)),
{"env":
{
"machines": machine_list,
"dns": dns,
"load-balance-ip": master_node["hostip"],
"service-cluster-ip-range": service_cluster_ip_range,
"api-servers-url": api_servers_url,
"dashboard-host": dashboard_host,
"dashboard-url": dashboard_url,
"pai-version": pai_version.paictl_version()
}
}))