def create(deployment_path: str, **kwargs):
# Late import.
import yaml
from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
from maro.cli.grass.executors.grass_local_executor import GrassLocalExecutor
from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor
from maro.utils.exception.cli_exception import BadRequestError, FileOperationError, InvalidDeploymentTemplateError
try:
with open(deployment_path, "r") as fr:
create_deployment = yaml.safe_load(fr)
if create_deployment["mode"] == "grass/azure":
GrassAzureExecutor.create(create_deployment=create_deployment)
elif create_deployment["mode"] == "grass/on-premises":
GrassOnPremisesExecutor.create(create_deployment=create_deployment)
elif create_deployment["mode"] == "grass/local":
executor = GrassLocalExecutor(
cluster_name=create_deployment["name"],
cluster_details=create_deployment)
executor.create()
else:
raise BadRequestError(
f"Unsupported operation in mode '{create_deployment['mode']}'."
)
except KeyError as e:
raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.")
except FileNotFoundError:
raise FileOperationError("Invalid template file path.")
def pull_data(self, local_dir: str, remote_path: str) -> None:
"""Pull remote AFS service data to local folder via azcopy.
Args:
local_dir (str): path of the local folder.
remote_path (str): path of the remote data.
Returns:
None.
"""
# Get sas
sas = self._check_and_get_account_sas()
# Push data
abs_local_dir = os.path.expanduser(local_dir)
source_path = PathConvertor.build_path_without_trailing_slash(
remote_path)
abs_target_dir = PathConvertor.build_path_with_trailing_slash(
abs_local_dir)
os.makedirs(abs_target_dir, exist_ok=True)
if not source_path.startswith("/"):
raise FileOperationError(
f"Invalid remote path: {source_path}\nShould be started with '/'"
)
copy_command = (
"azcopy copy "
f"'https://{self.cluster_id}st.file.core.windows.net/{self.cluster_id}-fs{source_path}?{sas}' "
f"'{abs_target_dir}' "
"--recursive=True")
_ = Subprocess.run(command=copy_command)
def create(deployment_path: str, **kwargs):
try:
with open(deployment_path, 'r') as fr:
create_deployment = yaml.safe_load(fr)
if create_deployment["mode"] == "k8s/aks":
K8sAksExecutor.build_cluster_details(
create_deployment=create_deployment)
executor = K8sAksExecutor(cluster_name=create_deployment["name"])
executor.create()
else:
raise BadRequestError(
f"Unsupported command in mode '{create_deployment['mode']}'.")
except KeyError as e:
raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.")
except FileNotFoundError:
raise FileOperationError("Invalid template file path.")
def node_join(node_join_path: str, **kwargs):
try:
with open(node_join_path, "r") as fr:
node_join_info = yaml.safe_load(fr)
fr.close()
if node_join_info["mode"] != "grass/on-premises":
raise BadRequestError(
f"Node join cluster interrupted: Invalid mode: {node_join_info['mode']}"
)
executor = GrassOnPremisesExecutor(node_join_info["cluster"])
executor.node_join_cluster(node_join_info)
except FileNotFoundError:
raise FileOperationError("Invalid template file path.")
def create(deployment_path: str, **kwargs):
# Late import.
import yaml
from maro.cli.k8s.executors.k8s_aks_executor import K8sAksExecutor
from maro.utils.exception.cli_exception import BadRequestError, FileOperationError, InvalidDeploymentTemplateError
try:
with open(deployment_path, "r") as fr:
create_deployment = yaml.safe_load(fr)
if create_deployment["mode"] == "k8s/aks":
K8sAksExecutor.create(create_deployment=create_deployment)
else:
raise BadRequestError(
f"Unsupported operation in mode '{create_deployment['mode']}'."
)
except KeyError as e:
raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.")
except FileNotFoundError:
raise FileOperationError("Invalid template file path.")
def copy_and_rename(source_path: str, target_dir: str, new_name: str = None):
"""Copy and rename a file.
Args:
source_path (str): path of the source
target_dir (str): dir of the target
new_name (str): name of the new file, if None, will not do rename
"""
source_path = os.path.expanduser(source_path)
target_dir = os.path.expanduser(target_dir)
if os.path.isdir(source_path):
raise FileOperationError(f"Cannot be a folder: '{source_path}'.")
shutil.copy2(source_path, target_dir)
if new_name is not None:
old_name = os.path.basename(source_path)
old_target_path = os.path.join(target_dir, old_name)
new_target_path = os.path.join(target_dir, new_name)
os.rename(old_target_path, new_target_path)
def pull_data(self, local_path: str, remote_path: str) -> None:
"""Pull data from remote MARO Cluster to local.
Args:
local_path (str): path of the local folder.
remote_path (str): path of the remote file.
Returns:
None.
"""
if not remote_path.startswith("/"):
raise FileOperationError(
f"Invalid remote path: {remote_path}\nShould be started with '/'"
)
FileSynchronizer.copy_files_from_node(
local_dir=local_path,
remote_path=
f"{GlobalPaths.MARO_SHARED}/clusters/{self.cluster_name}/data{remote_path}",
node_username=self.master_username,
node_hostname=self.master_public_ip_address,
node_ssh_port=self.master_ssh_port)
def join_cluster(deployment_path: str, **kwargs):
# Late import.
import yaml
from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor
from maro.utils.exception.cli_exception import BadRequestError, FileOperationError, InvalidDeploymentTemplateError
try:
with open(deployment_path, "r") as fr:
join_cluster_deployment = yaml.safe_load(stream=fr)
if join_cluster_deployment["mode"] == "grass/on-premises":
GrassOnPremisesExecutor.join_cluster(
join_cluster_deployment=join_cluster_deployment)
else:
raise BadRequestError(
f"Unsupported operation in mode '{join_cluster_deployment['mode']}'."
)
except KeyError as e:
raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.")
except FileNotFoundError:
raise FileOperationError("Invalid template file path.")