def _create_k8s_cluster(cluster_details: dict) -> None:
"""Create k8s cluster for the MARO Cluster.
Args:
cluster_details (dict): details of the MARO Cluster.
Returns:
None.
"""
logger.info("Creating k8s cluster")
# Create ARM parameters and start deployment
template_file_path = f"{K8sPaths.ABS_MARO_K8S_LIB}/modes/aks/create_aks_cluster/template.json"
parameters_file_path = (
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{cluster_details['name']}/parameters/create_aks_cluster.json"
)
ArmTemplateParameterBuilder.create_aks_cluster(
cluster_details=cluster_details, export_path=parameters_file_path)
AzureController.start_deployment(
resource_group=cluster_details["cloud"]["resource_group"],
deployment_name="aks_cluster",
template_file_path=template_file_path,
parameters_file_path=parameters_file_path)
# Attach ACR
AzureController.attach_acr(
resource_group=cluster_details["cloud"]["resource_group"],
aks_name=f"{cluster_details['id']}-aks",
acr_name=f"{cluster_details['id']}acr")
logger.info_green("K8s cluster is created")
def _create_vnet(cluster_details: dict) -> None:
"""Create vnet for the MARO Cluster.
Args:
cluster_details (dict): details of the MARO Cluster.
Returns:
None.
"""
logger.info("Creating vnet")
# Create ARM parameters and start deployment
template_file_path = f"{GrassPaths.ABS_MARO_GRASS_LIB}/modes/azure/create_vnet/template.json"
parameters_file_path = (
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{cluster_details['name']}/vnet/arm_create_vnet_parameters.json"
)
ArmTemplateParameterBuilder.create_vnet(
cluster_details=cluster_details, export_path=parameters_file_path)
AzureController.start_deployment(
resource_group=cluster_details["cloud"]["resource_group"],
deployment_name="vnet",
template_file_path=template_file_path,
parameters_file_path=parameters_file_path)
logger.info_green("Vnet is created")
def _create_master_vm(cluster_details: dict) -> None:
"""Create MARO Master VM.
Args:
cluster_details (dict): details of the MARO Cluster.
Returns:
None.
"""
logger.info("Creating Master VM")
# Build params
vm_name = f"{cluster_details['id']}-master-vm"
# Create ARM parameters and start deployment
template_file_path = f"{GrassPaths.ABS_MARO_GRASS_LIB}/modes/azure/create_master/template.json"
parameters_file_path = (
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{cluster_details['name']}"
f"/master/arm_create_master_parameters.json")
ArmTemplateParameterBuilder.create_master(
cluster_details=cluster_details,
node_size=cluster_details["master"]["node_size"],
export_path=parameters_file_path)
AzureController.start_deployment(
resource_group=cluster_details["cloud"]["resource_group"],
deployment_name="master",
template_file_path=template_file_path,
parameters_file_path=parameters_file_path)
# Get master IP addresses
ip_addresses = AzureController.list_ip_addresses(
resource_group=cluster_details["cloud"]["resource_group"],
vm_name=vm_name)
public_ip_address = ip_addresses[0]["virtualMachine"]["network"][
"publicIpAddresses"][0]["ipAddress"]
private_ip_address = ip_addresses[0]["virtualMachine"]["network"][
"privateIpAddresses"][0]
# Get other params and fill them to master_details
hostname = vm_name
username = cluster_details["cloud"]["default_username"]
cluster_details["master"]["hostname"] = hostname
cluster_details["master"]["username"] = username
cluster_details["master"]["public_ip_address"] = public_ip_address
cluster_details["master"]["private_ip_address"] = private_ip_address
cluster_details["master"]["resource_name"] = vm_name
cluster_details["master"]["ssh"] = {
"port": cluster_details["connection"]["ssh"]["port"]
}
cluster_details["master"]["api_server"] = {
"port": cluster_details["connection"]["api_server"]["port"]
}
logger.info_green(
f"You can login to your master node with: {username}@{public_ip_address}"
)
logger.info_green("Master VM is created")
def _create_virtual_machines(cls, test_config: dict):
cls.build_arm_parameters(
build_config={
"location": cls.location,
"default_username": cls.default_username,
"default_public_key": test_config["cloud/default_public_key"],
"ssh": {
"port": GlobalParams.DEFAULT_SSH_PORT
},
"api_server": {
"port": GrassParams.DEFAULT_API_SERVER_PORT
}
},
export_path=cls.arm_parameters_file_export_path)
AzureController.set_subscription(
subscription=test_config["cloud/subscription"])
AzureController.create_resource_group(
resource_group=cls.resource_group, location=cls.location)
AzureController.start_deployment(
resource_group=cls.resource_group,
deployment_name="cluster",
template_file_path=cls.arm_template_file_path,
parameters_file_path=cls.arm_parameters_file_export_path)
def setUpClass(cls) -> None:
# Get and set params
GlobalParams.LOG_LEVEL = logging.DEBUG
cls.test_id = uuid.uuid4().hex[:8]
os.makedirs(
os.path.expanduser(f"{GlobalPaths.MARO_TEST}/{cls.test_id}"),
exist_ok=True)
cls.test_file_path = os.path.abspath(__file__)
cls.test_dir_path = os.path.dirname(cls.test_file_path)
# Load config
cls.config_path = os.path.normpath(
os.path.join(cls.test_dir_path, "./config.yml"))
# Load config
with open(cls.config_path) as fr:
config_details = yaml.safe_load(fr)
if config_details["cloud/subscription"] and config_details[
"user/admin_public_key"]:
pass
else:
raise Exception("Invalid config")
# Create resource group
AzureController.create_resource_group(cls.resource_group, cls.location)
# Create ARM params
template_file_location = f"{cls.test_dir_path}/test_checkpoint_template.json"
base_parameters_file_location = f"{cls.test_dir_path}/test_checkpoint_parameters.json"
parameters_file_location = os.path.expanduser(
f"{GlobalPaths.MARO_TEST}/{cls.test_id}/test_checkpoint_parameters.json"
)
with open(base_parameters_file_location, "r") as f:
base_parameters = json.load(f)
with open(parameters_file_location, "w") as fw:
parameters = base_parameters["parameters"]
parameters["location"]["value"] = cls.location
parameters["networkInterfaceName"]["value"] = f"{cls.test_id}-nic"
parameters["networkSecurityGroupName"][
"value"] = f"{cls.test_id}-nsg"
parameters["virtualNetworkName"]["value"] = f"{cls.test_id}-vnet"
parameters["publicIpAddressName"]["value"] = f"{cls.test_id}-pip"
parameters["virtualMachineName"]["value"] = f"{cls.test_id}-vm"
parameters["virtualMachineSize"]["value"] = "Standard_B2s"
parameters["adminUsername"]["value"] = cls.admin_username
parameters["adminPublicKey"]["value"] = config_details[
"user/admin_public_key"]
parameters["storageAccountName"]["value"] = f"{cls.test_id}st"
json.dump(base_parameters, fw, indent=4)
# Start ARM deployment
AzureController.start_deployment(
resource_group=cls.resource_group,
deployment_name=cls.test_id,
template_file=template_file_location,
parameters_file=parameters_file_location)
cls._gracefully_wait(15)
# Get params after ARM deployment
cls.conn_str = AzureController.get_connection_string(
storage_account_name=f"{cls.test_id}st")
ip_addresses = AzureController.list_ip_addresses(
resource_group=cls.resource_group, vm_name=f"{cls.test_id}-vm")
cls.ip_address = ip_addresses[0]["virtualMachine"]["network"][
"publicIpAddresses"][0]["ipAddress"]
def _create_vm(self, node_name: str, node_size: str) -> dict:
"""Create MARO Node VM.
Args:
node_name (str): name of the MARO Node. Also the id of the MARO Node.
node_size (str): size of the MARO Node VM.
Returns:
dict: join_cluster_deployment that needed in "join cluster" operation.
See /lib/scripts/join_cluster.py for reference.
"""
logger.info(message=f"Creating VM '{node_name}'")
# Create ARM parameters and start deployment
os.makedirs(
name=
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{self.cluster_name}/nodes/{node_name}",
exist_ok=True)
template_file_path = f"{GrassPaths.ABS_MARO_GRASS_LIB}/modes/azure/create_node/template.json"
parameters_file_path = (
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{self.cluster_name}/nodes/{node_name}/arm_create_node_parameters.json"
)
ArmTemplateParameterBuilder.create_node(
node_name=node_name,
cluster_details=self.cluster_details,
node_size=node_size,
export_path=parameters_file_path)
AzureController.start_deployment(
resource_group=self.resource_group,
deployment_name=node_name,
template_file_path=template_file_path,
parameters_file_path=parameters_file_path)
# Get node IP addresses
ip_addresses = AzureController.list_ip_addresses(
resource_group=self.resource_group,
vm_name=f"{self.cluster_id}-{node_name}-vm")
logger.info_green(f"VM '{node_name}' is created")
# Build join_cluster_deployment.
join_cluster_deployment = {
"mode": "grass/azure",
"master": {
"private_ip_address": self.master_private_ip_address,
"api_server": {
"port": self.master_api_server_port
},
"redis": {
"port": self.master_redis_port
}
},
"node": {
"name":
node_name,
"id":
node_name,
"username":
self.default_username,
"public_ip_address":
ip_addresses[0]["virtualMachine"]["network"]
["publicIpAddresses"][0]["ipAddress"],
"private_ip_address":
ip_addresses[0]["virtualMachine"]["network"]
["privateIpAddresses"][0],
"node_size":
node_size,
"resource_name":
f"{self.cluster_id}-{node_name}-vm",
"hostname":
f"{self.cluster_id}-{node_name}-vm",
"resources": {
"cpu": "all",
"memory": "all",
"gpu": "all"
},
"api_server": {
"port": self.api_server_port
},
"ssh": {
"port": self.ssh_port
}
},
"configs": {
"install_node_runtime": False,
"install_node_gpu_support": False
}
}
with open(
file=
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{self.cluster_name}/nodes/{node_name}/join_cluster_deployment.yml",
mode="w") as fw:
yaml.safe_dump(data=join_cluster_deployment, stream=fw)
return join_cluster_deployment
def _build_node_image(cluster_details: dict) -> None:
"""Build Azure Image for MARO Node.
The built image will contain required Node runtime environment including GPU support.
See https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image for reference.
Args:
cluster_details (dict): details of the MARO Cluster.
Returns:
None.
"""
logger.info("Building MARO Node image")
# Build params
resource_name = "build-node-image"
image_name = f"{cluster_details['id']}-node-image"
vm_name = f"{cluster_details['id']}-{resource_name}-vm"
# Create ARM parameters and start deployment.
# For simplicity, we use master_node_size as the size of build_node_image_vm here
template_file_path = f"{GrassPaths.ABS_MARO_GRASS_LIB}/modes/azure/create_build_node_image_vm/template.json"
parameters_file_path = (
f"{GlobalPaths.ABS_MARO_CLUSTERS}/{cluster_details['name']}"
f"/build_node_image_vm/arm_create_build_node_image_vm_parameters.json"
)
ArmTemplateParameterBuilder.create_build_node_image_vm(
cluster_details=cluster_details,
node_size=cluster_details["master"]["node_size"],
export_path=parameters_file_path)
AzureController.start_deployment(
resource_group=cluster_details["cloud"]["resource_group"],
deployment_name=resource_name,
template_file_path=template_file_path,
parameters_file_path=parameters_file_path)
# Gracefully wait
time.sleep(10)
# Get public ip address
ip_addresses = AzureController.list_ip_addresses(
resource_group=cluster_details["cloud"]["resource_group"],
vm_name=vm_name)
public_ip_address = ip_addresses[0]["virtualMachine"]["network"][
"publicIpAddresses"][0]["ipAddress"]
# Make sure build_node_image_vm is able to connect
GrassAzureExecutor.retry_connection(
node_username=cluster_details["cloud"]["default_username"],
node_hostname=public_ip_address,
node_ssh_port=cluster_details["connection"]["ssh"]["port"])
# Run init image script
FileSynchronizer.copy_files_to_node(
local_path=
f"{GrassPaths.MARO_GRASS_LIB}/scripts/build_node_image_vm/init_build_node_image_vm.py",
remote_dir="~/",
node_username=cluster_details["cloud"]["default_username"],
node_hostname=public_ip_address,
node_ssh_port=cluster_details["connection"]["ssh"]["port"])
GrassAzureExecutor.remote_init_build_node_image_vm(
node_username=cluster_details["cloud"]["default_username"],
node_hostname=public_ip_address,
node_ssh_port=cluster_details["connection"]["ssh"]["port"])
# Extract image
AzureController.deallocate_vm(
resource_group=cluster_details["cloud"]["resource_group"],
vm_name=vm_name)
AzureController.generalize_vm(
resource_group=cluster_details["cloud"]["resource_group"],
vm_name=vm_name)
AzureController.create_image_from_vm(
resource_group=cluster_details["cloud"]["resource_group"],
image_name=image_name,
vm_name=vm_name)
# Delete resources
GrassAzureExecutor._delete_resources(
resource_group=cluster_details["cloud"]["resource_group"],
resource_name=resource_name,
cluster_id=cluster_details["id"])
logger.info_green("MARO Node Image is built")