def main():
parser = argparse.ArgumentParser()
parser.add_argument('-l',
'--layout',
dest="layout",
required=True,
help="layout.yaml")
parser.add_argument('-c',
'--config',
dest="config",
required=True,
help="cluster configuration")
parser.add_argument('-o',
'--output',
dest="output",
required=True,
help="cluster configuration")
args = parser.parse_args()
output_path = os.path.expanduser(args.output)
layout = load_yaml_config(args.layout)
config = load_yaml_config(args.config)
masters, workers = get_masters_workers_from_layout(layout)
head_node = masters[0]
# fill in cpu, memory, computing_device information in both masters and workers
# we assume the layout file the user gives is correct
all_machines = masters + workers
for machine in all_machines:
sku_info = layout['machine-sku'][machine['machine-type']]
# use math.ceil to guarantee the memory volume
# e.g. if use set 999.1MB, we ensure there is 1000MB to avoid scheduling issues
machine['memory_mb'] = math.ceil(
parse_quantity(sku_info['mem']) / 1024 / 1024)
machine['cpu_vcores'] = sku_info['cpu']['vcore']
if 'computing-device' in sku_info:
machine['computing_device'] = sku_info['computing-device']
# add machine to different comupting device group
computing_device_groups = defaultdict(list)
for machine in all_machines:
sku_info = layout['machine-sku'][machine['machine-type']]
if 'computing-device' in sku_info:
computing_device_groups[sku_info['computing-device']
['type']].append(machine['hostname'])
environment = {
'masters': masters,
'workers': workers,
'cfg': config,
'head_node': head_node,
'computing_device_groups': computing_device_groups,
}
map_table = {"env": environment}
generate_template_file("quick-start/pre-check.yml.template",
"{0}/pre-check.yml".format(output_path), map_table)
def check_layout(layout, cluster_config):
# hostname / hostip should be unique
hostnames = [elem['hostname'] for elem in layout['machine-list']]
if len(hostnames) != len(set(hostnames)):
logger.error("hostname should be unique")
return False
hostips = [elem['hostip'] for elem in layout['machine-list']]
if len(hostips) != len(set(hostips)):
logger.error("hostip should be unique")
return False
# machine-type should be defined in machine-sku
# collect types of computing device
worker_computing_devices = set()
for machine in layout['machine-list']:
if machine['machine-type'] not in layout['machine-sku']:
logger.error("machine-type %s is not defined",
machine['machine-type'])
return False
machine_sku = layout['machine-sku'][machine['machine-type']]
if 'pai-worker' in machine and machine[
'pai-worker'] == 'true' and 'computing-device' in machine_sku:
worker_computing_devices.add(
machine_sku['computing-device']['type'])
worker_computing_devices = list(worker_computing_devices)
masters, workers = get_masters_workers_from_layout(layout)
# only one pai-master
if len(masters) == 0:
logger.error('No master machine specified.')
return False
if len(masters) > 1:
logger.error('More than one master machine specified.')
return False
# at least one pai-worker
if len(workers) == 0:
logger.error('No worker machine specified.')
return False
# pai-master / pai-worker cannot be true at the same time
if 'pai-worker' in masters[0] and masters[0]['pai-worker'] == 'true':
logger.error(
"One machine can not be pai-master and pai-worker at the same time."
)
return False
# if cluster_config.enable_hived_scheduler is false, there should be <= 1 type of computing device
if 'enable_hived_scheduler' in cluster_config and cluster_config[
'enable_hived_scheduler'] is False:
if len(worker_computing_devices) > 1:
logger.error(
'K8S default scheduler only supports <= 1 type of computing device in worker nodes.'
)
return False
return True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-l',
'--layout',
dest="layout",
required=True,
help="layout.yaml")
parser.add_argument('-c',
'--config',
dest="config",
required=True,
help="cluster configuration")
parser.add_argument('-o',
'--output',
dest="output",
required=True,
help="cluster configuration")
args = parser.parse_args()
output_path = os.path.expanduser(args.output)
layout = load_yaml_config(args.layout)
cluster_config = load_yaml_config(args.config)
masters, workers = get_masters_workers_from_layout(layout)
head_node = masters[0]
# Hivedscheduler is enabled by default.
# But if the user sets enable_hived_scheduler to false manually,
# we should disable it.
if 'enable_hived_scheduler' in cluster_config and cluster_config[
'enable_hived_scheduler'] is False:
hived_config = {}
else:
hived_config = get_hived_config(layout, cluster_config)
environment = {
'masters': masters,
'workers': workers,
'cfg': cluster_config,
'head_node': head_node,
'hived': hived_config
}
map_table = {"env": environment}
generate_template_file(
os.path.abspath(
os.path.join(
os.path.abspath(__file__),
'../../quick-start/services-configuration.yaml.template')),
"{0}/services-configuration.yaml".format(output_path), map_table)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-l', '--layout', dest="layout", required=True,
help="layout.yaml")
parser.add_argument('-c', '--config', dest="config", required=True,
help="cluster configuration")
parser.add_argument('-o', '--output', dest="output", required=True,
help="cluster configuration")
args = parser.parse_args()
output_path = os.path.expanduser(args.output)
layout = load_yaml_config(args.layout)
cluster_config = load_yaml_config(args.config)
masters, workers = get_masters_workers_from_layout(layout)
head_node = masters[0]
environment = {
'masters': masters,
'workers': workers,
'cfg': cluster_config,
'head_node': head_node
}
map_table = {
"env": environment
}
generate_template_file(
"quick-start/hosts.yml.template",
"{0}/hosts.yml".format(output_path),
map_table
)
generate_template_file(
"quick-start/prophet.yml.template",
"{0}/prophet.yml".format(output_path),
map_table
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-l',
'--layout',
dest="layout",
required=True,
help="layout.yaml")
parser.add_argument('-c',
'--config',
dest="config",
required=True,
help="cluster configuration")
parser.add_argument('-o',
'--output',
dest="output",
required=True,
help="cluster configuration")
args = parser.parse_args()
output_path = os.path.expanduser(args.output)
layout = load_yaml_config(args.layout)
cluster_config = load_yaml_config(args.config)
masters, workers = get_masters_workers_from_layout(layout)
head_node = masters[0]
if 'openpai_kube_network_plugin' not in cluster_config or cluster_config[
'openpai_kube_network_plugin'] != 'weave':
count_input = 0
while True:
user_input = input(
"Are your cluster is in Azure cloud or not? (Y/N) (case sensitive)"
)
if user_input == "N":
break
if user_input == "Y":
break
print(" Please type Y or N. It's case sensitive.")
count_input = count_input + 1
if count_input == 3:
logger.error(
"3 Times......... Sorry, we will force stopping your operation."
)
sys.exit(1)
if user_input == "Y" \
and ('openpai_kube_network_plugin' not in cluster_config or cluster_config['openpai_kube_network_plugin'] == 'calico'):
logger.error(
"Azure does not support calico, please change the openpai_kube_network_plugin to weave"
)
logger.error(
"https://docs.projectcalico.org/reference/public-cloud/azure#why-doesnt-azure-support-calico-networking"
)
sys.exit(1)
environment = {
'masters': masters,
'workers': workers,
'cfg': cluster_config,
'head_node': head_node
}
map_table = {"env": environment}
generate_template_file("quick-start/hosts.yml.template",
"{0}/hosts.yml".format(output_path), map_table)
generate_template_file("quick-start/openpai.yml.template",
"{0}/openpai.yml".format(output_path), map_table)