def create(deployment_path: str, **kwargs): # Late import. import yaml from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor from maro.cli.grass.executors.grass_local_executor import GrassLocalExecutor from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor from maro.utils.exception.cli_exception import BadRequestError, FileOperationError, InvalidDeploymentTemplateError try: with open(deployment_path, "r") as fr: create_deployment = yaml.safe_load(fr) if create_deployment["mode"] == "grass/azure": GrassAzureExecutor.create(create_deployment=create_deployment) elif create_deployment["mode"] == "grass/on-premises": GrassOnPremisesExecutor.create(create_deployment=create_deployment) elif create_deployment["mode"] == "grass/local": executor = GrassLocalExecutor( cluster_name=create_deployment["name"], cluster_details=create_deployment) executor.create() else: raise BadRequestError( f"Unsupported operation in mode '{create_deployment['mode']}'." ) except KeyError as e: raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.") except FileNotFoundError: raise FileOperationError("Invalid template file path.")
def pull_data(self, local_dir: str, remote_path: str) -> None: """Pull remote AFS service data to local folder via azcopy. Args: local_dir (str): path of the local folder. remote_path (str): path of the remote data. Returns: None. """ # Get sas sas = self._check_and_get_account_sas() # Push data abs_local_dir = os.path.expanduser(local_dir) source_path = PathConvertor.build_path_without_trailing_slash( remote_path) abs_target_dir = PathConvertor.build_path_with_trailing_slash( abs_local_dir) os.makedirs(abs_target_dir, exist_ok=True) if not source_path.startswith("/"): raise FileOperationError( f"Invalid remote path: {source_path}\nShould be started with '/'" ) copy_command = ( "azcopy copy " f"'https://{self.cluster_id}st.file.core.windows.net/{self.cluster_id}-fs{source_path}?{sas}' " f"'{abs_target_dir}' " "--recursive=True") _ = Subprocess.run(command=copy_command)
def create(deployment_path: str, **kwargs): try: with open(deployment_path, 'r') as fr: create_deployment = yaml.safe_load(fr) if create_deployment["mode"] == "k8s/aks": K8sAksExecutor.build_cluster_details( create_deployment=create_deployment) executor = K8sAksExecutor(cluster_name=create_deployment["name"]) executor.create() else: raise BadRequestError( f"Unsupported command in mode '{create_deployment['mode']}'.") except KeyError as e: raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.") except FileNotFoundError: raise FileOperationError("Invalid template file path.")
def node_join(node_join_path: str, **kwargs): try: with open(node_join_path, "r") as fr: node_join_info = yaml.safe_load(fr) fr.close() if node_join_info["mode"] != "grass/on-premises": raise BadRequestError( f"Node join cluster interrupted: Invalid mode: {node_join_info['mode']}" ) executor = GrassOnPremisesExecutor(node_join_info["cluster"]) executor.node_join_cluster(node_join_info) except FileNotFoundError: raise FileOperationError("Invalid template file path.")
def create(deployment_path: str, **kwargs): # Late import. import yaml from maro.cli.k8s.executors.k8s_aks_executor import K8sAksExecutor from maro.utils.exception.cli_exception import BadRequestError, FileOperationError, InvalidDeploymentTemplateError try: with open(deployment_path, "r") as fr: create_deployment = yaml.safe_load(fr) if create_deployment["mode"] == "k8s/aks": K8sAksExecutor.create(create_deployment=create_deployment) else: raise BadRequestError( f"Unsupported operation in mode '{create_deployment['mode']}'." ) except KeyError as e: raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.") except FileNotFoundError: raise FileOperationError("Invalid template file path.")
def copy_and_rename(source_path: str, target_dir: str, new_name: str = None): """Copy and rename a file. Args: source_path (str): path of the source target_dir (str): dir of the target new_name (str): name of the new file, if None, will not do rename """ source_path = os.path.expanduser(source_path) target_dir = os.path.expanduser(target_dir) if os.path.isdir(source_path): raise FileOperationError(f"Cannot be a folder: '{source_path}'.") shutil.copy2(source_path, target_dir) if new_name is not None: old_name = os.path.basename(source_path) old_target_path = os.path.join(target_dir, old_name) new_target_path = os.path.join(target_dir, new_name) os.rename(old_target_path, new_target_path)
def pull_data(self, local_path: str, remote_path: str) -> None: """Pull data from remote MARO Cluster to local. Args: local_path (str): path of the local folder. remote_path (str): path of the remote file. Returns: None. """ if not remote_path.startswith("/"): raise FileOperationError( f"Invalid remote path: {remote_path}\nShould be started with '/'" ) FileSynchronizer.copy_files_from_node( local_dir=local_path, remote_path= f"{GlobalPaths.MARO_SHARED}/clusters/{self.cluster_name}/data{remote_path}", node_username=self.master_username, node_hostname=self.master_public_ip_address, node_ssh_port=self.master_ssh_port)
def join_cluster(deployment_path: str, **kwargs): # Late import. import yaml from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor from maro.utils.exception.cli_exception import BadRequestError, FileOperationError, InvalidDeploymentTemplateError try: with open(deployment_path, "r") as fr: join_cluster_deployment = yaml.safe_load(stream=fr) if join_cluster_deployment["mode"] == "grass/on-premises": GrassOnPremisesExecutor.join_cluster( join_cluster_deployment=join_cluster_deployment) else: raise BadRequestError( f"Unsupported operation in mode '{join_cluster_deployment['mode']}'." ) except KeyError as e: raise InvalidDeploymentTemplateError(f"Missing key '{e.args[0]}'.") except FileNotFoundError: raise FileOperationError("Invalid template file path.")