def push_data(self, local_path: str, remote_dir: str): # Load details cluster_details = self.cluster_details cluster_id = cluster_details['id'] # Get sas sas = self._check_and_get_account_sas() # Push data local_path = os.path.expanduser(local_path) source_path = get_reformatted_source_path(local_path) target_dir = get_reformatted_target_dir(remote_dir) if not target_dir.startswith("/"): raise CliException("Invalid remote path") copy_command = f'azcopy copy ' \ f'"{source_path}" ' \ f'"https://{cluster_id}st.file.core.windows.net/{cluster_id}-fs{target_dir}?{sas}" ' \ f'--recursive=True' _ = SubProcess.run(copy_command)
def test_2_azcopy_tar_big_file_to_local(self) -> None: # create folder command = f"mkdir -p {GlobalPaths.MARO_TEST}/{self.test_id}/test_2_azcopy_tar_big_file_to_local" SubProcess.interactive_run(command) # remote tar zip basename = os.path.basename( f"/mnt/maro/{self.test_id}/test_1_azcopy_tar_big_file_to_remote") dirname = os.path.dirname( f"/mnt/maro/{self.test_id}/test_1_azcopy_tar_big_file_to_remote") tar_file_name = uuid.uuid4().hex[:8] command = ( f"kubectl exec -i {self.pod_name} -- " f"tar cf /mnt/maro/tar/{tar_file_name} -C {dirname} {basename}") SubProcess.interactive_run(command) # azcopy sas = self.executor._check_and_get_account_sas() local_path = os.path.expanduser( f"{GlobalPaths.MARO_TEST}/{self.test_id}/tar/{tar_file_name}") command = ( f"azcopy copy " f"'https://{self.cluster_id}st.file.core.windows.net/{self.cluster_id}-fs" f"/tar/{tar_file_name}?{sas}' " f"'{local_path}' " f"--recursive=True") SubProcess.interactive_run(command) # local tar unzip command = ( f"tar xf {GlobalPaths.MARO_TEST}/{self.test_id}/tar/{tar_file_name} " f"-C {GlobalPaths.MARO_TEST}/{self.test_id}/test_2_azcopy_tar_big_file_to_local" ) SubProcess.interactive_run(command) self.assertTrue( os.path.exists( os.path.expanduser( f"{GlobalPaths.MARO_TEST}/{self.test_id}/" f"test_2_azcopy_tar_big_file_to_local/test_1_azcopy_tar_big_file_to_remote/big_file" )))
def copy_files_from_node(local_dir: str, remote_path: str, admin_username: str, node_ip_address: str, ssh_port: int) -> None: """Copy node files to local, automatically create folder if not exist. Args: local_dir (str): dir for local files remote_path (str): path of the remote file admin_username (str) node_ip_address (str) ssh_port (int): port of the ssh connection """ source_path = get_reformatted_source_path(remote_path) basename = os.path.basename(source_path) folder_name = os.path.dirname(source_path) target_dir = get_reformatted_target_dir(local_dir) # Create local dir os.makedirs(os.path.expanduser(target_dir), exist_ok=True) if platform.system() in ["Linux", "Darwin"]: # Copy with pipe copy_script = ( f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} " f"'tar czf - -C {folder_name} {basename}' | tar xzf - -C {target_dir}" ) _ = SubProcess.run(copy_script) else: # Copy with tmp file tmp_file_name = uuid.uuid4() maro_local_tmp_abs_path = os.path.expanduser( GlobalPaths.MARO_LOCAL_TMP) tar_script = ( f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} " f"tar czf {GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name} -C {folder_name} {basename}" ) _ = SubProcess.run(tar_script) copy_script = ( f"scp {admin_username}@{node_ip_address}:{GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name} " f"{maro_local_tmp_abs_path}") _ = SubProcess.run(copy_script) untar_script = f"tar xzf {maro_local_tmp_abs_path}/{tmp_file_name} -C {os.path.expanduser(target_dir)}" _ = SubProcess.run(untar_script) remove_script = f"rm {maro_local_tmp_abs_path}/{tmp_file_name}" _ = SubProcess.run(remove_script) remote_remove_script = ( f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} " f"'rm {GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name}'") _ = SubProcess.run(remote_remove_script)
def _delete_node(self, node_name: str): logger.info(f"Deleting node {node_name}") # Load details cluster_details = self.cluster_details cluster_id = cluster_details['id'] resource_group = cluster_details['cloud']['resource_group'] # Get resource list resource_list = AzureExecutor.list_resources(resource_group=resource_group) # Filter resources deletable_ids = [] for resource_info in resource_list: if resource_info['name'].startswith(f"{cluster_id}-{node_name}"): deletable_ids.append(resource_info['id']) # Delete resources if len(deletable_ids) > 0: AzureExecutor.delete_resources(resources=deletable_ids) # Delete azure deployment AzureExecutor.delete_deployment( resource_group=resource_group, deployment_name=node_name ) # Delete parameters_file parameters_file_location = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/parameters/{node_name}.json" command = f"rm {parameters_file_location}" _ = SubProcess.run(command) # Update node status self.grass_executor.remote_update_node_status( node_name=node_name, action='delete' ) logger.info_green(f"Node {node_name} is deleted")
def status(self): return_status = {} # Get pods details command = "kubectl get pods -o json" return_str = SubProcess.run(command) pods_details = json.loads(return_str)['items'] for pod_details in pods_details: if pod_details['metadata']['labels']['app'] == 'maro-redis': return_status['redis'] = { 'private_ip_address': pod_details['status']['podIP'] } break # Print status logger.info( json.dumps( return_status, indent=4, sort_keys=True ) )
def _start_job(self, job_details: dict): # Validate and fill optional value to deployment K8sAzureExecutor._standardize_start_job_deployment( start_job_deployment=job_details) job_name = job_details['name'] # Mkdir and save job details os.makedirs(os.path.expanduser( f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}" ), exist_ok=True) save_job_details(cluster_name=self.cluster_name, job_name=job_name, job_details=job_details) # Set job id self._set_job_id(job_name=job_name) # Create folder os.makedirs(os.path.expanduser( f'{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs' ), exist_ok=True) # Create and save k8s config k8s_job_config = self._create_k8s_job_config(job_name=job_name) with open( os.path.expanduser( f'{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs/jobs.yml' ), 'w') as fw: yaml.safe_dump(k8s_job_config, fw) # Apply k8s config command = f"kubectl apply -f " \ f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs/jobs.yml" _ = SubProcess.run(command)
def get_storage_account_keys(resource_group: str, storage_account_name: str): command = f'az storage account keys list -g {resource_group} --account-name {storage_account_name}' return_str = SubProcess.run(command) return json.loads(return_str)
def list_acr_repositories(acr_name: str): command = f"az acr repository list -n {acr_name}" return_str = SubProcess.run(command) return json.loads(return_str)
def login_acr(acr_name: str): command = f"az acr login --name {acr_name}" _ = SubProcess.run(command)
def list_nodepool(resource_group: str, aks_name: str): command = f'az aks nodepool list -g {resource_group} --cluster-name {aks_name}' return_str = SubProcess.run(command) return json.loads(return_str)
def list_vm_sizes(location: str): command = f"az vm list-sizes -l {location}" return_str = SubProcess.run(command) return json.loads(return_str)
def delete_resources(resources: list): command = f"az resource delete --ids {' '.join(resources)}" _ = SubProcess.run(command)
def generalize_vm(resource_group: str, vm_name: str) -> None: command = f"az vm generalize --resource-group {resource_group} --name {vm_name}" _ = SubProcess.run(command)
def _export_log(pod_id: str, container_name: str, export_dir: str): os.makedirs(os.path.expanduser(export_dir + f"/{pod_id}"), exist_ok=True) with open(os.path.expanduser(export_dir + f"/{pod_id}/{container_name}.log"), "w") as fw: command = f"kubectl logs {pod_id} {container_name}" return_str = SubProcess.run(command) fw.write(return_str)
def create_resource_group(resource_group: str, location: str): command = f"az group create --name {resource_group} --location {location}" _ = SubProcess.run(command)
def stop_job(self, job_name: str): # Stop job command = f"kubectl delete -f " \ f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs/jobs.yml" _ = SubProcess.run(command)
def get_image_resource_id(resource_group: str, image_name: str) -> str: command = f"az image show --resource-group {resource_group} --name {image_name}" return_str = SubProcess.run(command) return json.loads(return_str)["id"]
def create_image_from_vm(resource_group: str, image_name: str, vm_name: str) -> None: command = f"az image create --resource-group {resource_group} --name {image_name} --source {vm_name}" _ = SubProcess.run(command)
def set_subscription(subscription: str): command = f"az account set --subscription {subscription}" _ = SubProcess.run(command)
def list_ip_addresses(resource_group: str, vm_name: str): command = f"az vm list-ip-addresses -g {resource_group} --name {vm_name}" return_str = SubProcess.run(command) return json.loads(return_str)
def get_version(): command = "az version" return_str = SubProcess.run(command) return json.loads(return_str)
def deallocate_vm(resource_group: str, vm_name: str) -> None: command = f"az vm deallocate --resource-group {resource_group} --name {vm_name}" _ = SubProcess.run(command)
def list_resources(resource_group: str): command = f"az resource list -g {resource_group}" return_str = SubProcess.run(command) return json.loads(return_str)
def template(export_path: str): command = f'cp {GlobalPaths.MARO_K8S_LIB}/deployments/external/* {export_path}' _ = SubProcess.run(command)
def delete_deployment(resource_group: str, deployment_name: str): command = f"az deployment group delete -g {resource_group} --name {deployment_name}" _ = SubProcess.run(command)
def load_aks_context(resource_group: str, aks_name: str): command = f'az aks get-credentials -g {resource_group} --name {aks_name}' _ = SubProcess.run(command)
def stop_vm(resource_group: str, vm_name: str): command = f"az vm stop -g {resource_group} --name {vm_name}" _ = SubProcess.run(command)
def get_aks(resource_group: str, aks_name: str): command = f"az aks show -g {resource_group} -n {aks_name}" return_str = SubProcess.run(command) return json.loads(return_str)
def list_skus(vm_size: str, location: str): command = f"az vm list-skus -l {location} --all --size {vm_size}" return_str = SubProcess.run(command) return json.loads(return_str)
def attach_acr(resource_group: str, aks_name: str, acr_name: str): command = f'az aks update -g {resource_group} --name {aks_name} --attach-acr {acr_name}' _ = SubProcess.run(command)