def push_data(self, local_path: str, remote_dir: str):
# Load details
cluster_details = self.cluster_details
cluster_id = cluster_details['id']
# Get sas
sas = self._check_and_get_account_sas()
# Push data
local_path = os.path.expanduser(local_path)
source_path = get_reformatted_source_path(local_path)
target_dir = get_reformatted_target_dir(remote_dir)
if not target_dir.startswith("/"):
raise CliException("Invalid remote path")
copy_command = f'azcopy copy ' \
f'"{source_path}" ' \
f'"https://{cluster_id}st.file.core.windows.net/{cluster_id}-fs{target_dir}?{sas}" ' \
f'--recursive=True'
_ = SubProcess.run(copy_command)
def test_2_azcopy_tar_big_file_to_local(self) -> None:
# create folder
command = f"mkdir -p {GlobalPaths.MARO_TEST}/{self.test_id}/test_2_azcopy_tar_big_file_to_local"
SubProcess.interactive_run(command)
# remote tar zip
basename = os.path.basename(
f"/mnt/maro/{self.test_id}/test_1_azcopy_tar_big_file_to_remote")
dirname = os.path.dirname(
f"/mnt/maro/{self.test_id}/test_1_azcopy_tar_big_file_to_remote")
tar_file_name = uuid.uuid4().hex[:8]
command = (
f"kubectl exec -i {self.pod_name} -- "
f"tar cf /mnt/maro/tar/{tar_file_name} -C {dirname} {basename}")
SubProcess.interactive_run(command)
# azcopy
sas = self.executor._check_and_get_account_sas()
local_path = os.path.expanduser(
f"{GlobalPaths.MARO_TEST}/{self.test_id}/tar/{tar_file_name}")
command = (
f"azcopy copy "
f"'https://{self.cluster_id}st.file.core.windows.net/{self.cluster_id}-fs"
f"/tar/{tar_file_name}?{sas}' "
f"'{local_path}' "
f"--recursive=True")
SubProcess.interactive_run(command)
# local tar unzip
command = (
f"tar xf {GlobalPaths.MARO_TEST}/{self.test_id}/tar/{tar_file_name} "
f"-C {GlobalPaths.MARO_TEST}/{self.test_id}/test_2_azcopy_tar_big_file_to_local"
)
SubProcess.interactive_run(command)
self.assertTrue(
os.path.exists(
os.path.expanduser(
f"{GlobalPaths.MARO_TEST}/{self.test_id}/"
f"test_2_azcopy_tar_big_file_to_local/test_1_azcopy_tar_big_file_to_remote/big_file"
)))
def copy_files_from_node(local_dir: str, remote_path: str, admin_username: str,
node_ip_address: str, ssh_port: int) -> None:
"""Copy node files to local, automatically create folder if not exist.
Args:
local_dir (str): dir for local files
remote_path (str): path of the remote file
admin_username (str)
node_ip_address (str)
ssh_port (int): port of the ssh connection
"""
source_path = get_reformatted_source_path(remote_path)
basename = os.path.basename(source_path)
folder_name = os.path.dirname(source_path)
target_dir = get_reformatted_target_dir(local_dir)
# Create local dir
os.makedirs(os.path.expanduser(target_dir), exist_ok=True)
if platform.system() in ["Linux", "Darwin"]:
# Copy with pipe
copy_script = (
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"'tar czf - -C {folder_name} {basename}' | tar xzf - -C {target_dir}"
)
_ = SubProcess.run(copy_script)
else:
# Copy with tmp file
tmp_file_name = uuid.uuid4()
maro_local_tmp_abs_path = os.path.expanduser(
GlobalPaths.MARO_LOCAL_TMP)
tar_script = (
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"tar czf {GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name} -C {folder_name} {basename}"
)
_ = SubProcess.run(tar_script)
copy_script = (
f"scp {admin_username}@{node_ip_address}:{GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name} "
f"{maro_local_tmp_abs_path}")
_ = SubProcess.run(copy_script)
untar_script = f"tar xzf {maro_local_tmp_abs_path}/{tmp_file_name} -C {os.path.expanduser(target_dir)}"
_ = SubProcess.run(untar_script)
remove_script = f"rm {maro_local_tmp_abs_path}/{tmp_file_name}"
_ = SubProcess.run(remove_script)
remote_remove_script = (
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"'rm {GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name}'")
_ = SubProcess.run(remote_remove_script)
def _delete_node(self, node_name: str):
logger.info(f"Deleting node {node_name}")
# Load details
cluster_details = self.cluster_details
cluster_id = cluster_details['id']
resource_group = cluster_details['cloud']['resource_group']
# Get resource list
resource_list = AzureExecutor.list_resources(resource_group=resource_group)
# Filter resources
deletable_ids = []
for resource_info in resource_list:
if resource_info['name'].startswith(f"{cluster_id}-{node_name}"):
deletable_ids.append(resource_info['id'])
# Delete resources
if len(deletable_ids) > 0:
AzureExecutor.delete_resources(resources=deletable_ids)
# Delete azure deployment
AzureExecutor.delete_deployment(
resource_group=resource_group,
deployment_name=node_name
)
# Delete parameters_file
parameters_file_location = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/parameters/{node_name}.json"
command = f"rm {parameters_file_location}"
_ = SubProcess.run(command)
# Update node status
self.grass_executor.remote_update_node_status(
node_name=node_name,
action='delete'
)
logger.info_green(f"Node {node_name} is deleted")
def status(self):
return_status = {}
# Get pods details
command = "kubectl get pods -o json"
return_str = SubProcess.run(command)
pods_details = json.loads(return_str)['items']
for pod_details in pods_details:
if pod_details['metadata']['labels']['app'] == 'maro-redis':
return_status['redis'] = {
'private_ip_address': pod_details['status']['podIP']
}
break
# Print status
logger.info(
json.dumps(
return_status,
indent=4, sort_keys=True
)
)
def _start_job(self, job_details: dict):
# Validate and fill optional value to deployment
K8sAzureExecutor._standardize_start_job_deployment(
start_job_deployment=job_details)
job_name = job_details['name']
# Mkdir and save job details
os.makedirs(os.path.expanduser(
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}"
),
exist_ok=True)
save_job_details(cluster_name=self.cluster_name,
job_name=job_name,
job_details=job_details)
# Set job id
self._set_job_id(job_name=job_name)
# Create folder
os.makedirs(os.path.expanduser(
f'{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs'
),
exist_ok=True)
# Create and save k8s config
k8s_job_config = self._create_k8s_job_config(job_name=job_name)
with open(
os.path.expanduser(
f'{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs/jobs.yml'
), 'w') as fw:
yaml.safe_dump(k8s_job_config, fw)
# Apply k8s config
command = f"kubectl apply -f " \
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs/jobs.yml"
_ = SubProcess.run(command)
def get_storage_account_keys(resource_group: str,
storage_account_name: str):
command = f'az storage account keys list -g {resource_group} --account-name {storage_account_name}'
return_str = SubProcess.run(command)
return json.loads(return_str)
def list_acr_repositories(acr_name: str):
command = f"az acr repository list -n {acr_name}"
return_str = SubProcess.run(command)
return json.loads(return_str)
def login_acr(acr_name: str):
command = f"az acr login --name {acr_name}"
_ = SubProcess.run(command)
def list_nodepool(resource_group: str, aks_name: str):
command = f'az aks nodepool list -g {resource_group} --cluster-name {aks_name}'
return_str = SubProcess.run(command)
return json.loads(return_str)
def list_vm_sizes(location: str):
command = f"az vm list-sizes -l {location}"
return_str = SubProcess.run(command)
return json.loads(return_str)
def delete_resources(resources: list):
command = f"az resource delete --ids {' '.join(resources)}"
_ = SubProcess.run(command)
def generalize_vm(resource_group: str, vm_name: str) -> None:
command = f"az vm generalize --resource-group {resource_group} --name {vm_name}"
_ = SubProcess.run(command)
def _export_log(pod_id: str, container_name: str, export_dir: str):
os.makedirs(os.path.expanduser(export_dir + f"/{pod_id}"), exist_ok=True)
with open(os.path.expanduser(export_dir + f"/{pod_id}/{container_name}.log"), "w") as fw:
command = f"kubectl logs {pod_id} {container_name}"
return_str = SubProcess.run(command)
fw.write(return_str)
def create_resource_group(resource_group: str, location: str):
command = f"az group create --name {resource_group} --location {location}"
_ = SubProcess.run(command)
def stop_job(self, job_name: str):
# Stop job
command = f"kubectl delete -f " \
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/k8s_configs/jobs.yml"
_ = SubProcess.run(command)
def get_image_resource_id(resource_group: str, image_name: str) -> str:
command = f"az image show --resource-group {resource_group} --name {image_name}"
return_str = SubProcess.run(command)
return json.loads(return_str)["id"]
def create_image_from_vm(resource_group: str, image_name: str, vm_name: str) -> None:
command = f"az image create --resource-group {resource_group} --name {image_name} --source {vm_name}"
_ = SubProcess.run(command)
def set_subscription(subscription: str):
command = f"az account set --subscription {subscription}"
_ = SubProcess.run(command)
def list_ip_addresses(resource_group: str, vm_name: str):
command = f"az vm list-ip-addresses -g {resource_group} --name {vm_name}"
return_str = SubProcess.run(command)
return json.loads(return_str)
def get_version():
command = "az version"
return_str = SubProcess.run(command)
return json.loads(return_str)
def deallocate_vm(resource_group: str, vm_name: str) -> None:
command = f"az vm deallocate --resource-group {resource_group} --name {vm_name}"
_ = SubProcess.run(command)
def list_resources(resource_group: str):
command = f"az resource list -g {resource_group}"
return_str = SubProcess.run(command)
return json.loads(return_str)
def template(export_path: str):
command = f'cp {GlobalPaths.MARO_K8S_LIB}/deployments/external/* {export_path}'
_ = SubProcess.run(command)
def delete_deployment(resource_group: str, deployment_name: str):
command = f"az deployment group delete -g {resource_group} --name {deployment_name}"
_ = SubProcess.run(command)
def load_aks_context(resource_group: str, aks_name: str):
command = f'az aks get-credentials -g {resource_group} --name {aks_name}'
_ = SubProcess.run(command)
def stop_vm(resource_group: str, vm_name: str):
command = f"az vm stop -g {resource_group} --name {vm_name}"
_ = SubProcess.run(command)
def get_aks(resource_group: str, aks_name: str):
command = f"az aks show -g {resource_group} -n {aks_name}"
return_str = SubProcess.run(command)
return json.loads(return_str)
def list_skus(vm_size: str, location: str):
command = f"az vm list-skus -l {location} --all --size {vm_size}"
return_str = SubProcess.run(command)
return json.loads(return_str)
def attach_acr(resource_group: str, aks_name: str, acr_name: str):
command = f'az aks update -g {resource_group} --name {aks_name} --attach-acr {acr_name}'
_ = SubProcess.run(command)
