def copy_files_to_node(local_path: str, remote_dir: str, admin_username: str,
node_ip_address: str, ssh_port: int) -> None:
"""Copy local files to node, automatically create folder if not exist.
Args:
local_path (str): path of the local file
remote_dir (str): dir for remote files
admin_username (str)
node_ip_address (str)
ssh_port (int): port of the ssh connection
"""
source_path = get_reformatted_source_path(local_path)
basename = os.path.basename(source_path)
folder_name = os.path.expanduser(os.path.dirname(source_path))
target_dir = get_reformatted_target_dir(remote_dir)
mkdir_script = (
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"'mkdir -p {target_dir}'")
_ = SubProcess.run(mkdir_script)
if platform.system() in ["Linux", "Darwin"]:
# Copy with pipe
copy_script = (
f"tar czf - -C {folder_name} {basename} | "
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"'tar xzf - -C {target_dir}'")
_ = SubProcess.run(copy_script)
else:
# Copy with tmp file
tmp_file_name = uuid.uuid4()
maro_local_tmp_abs_path = os.path.expanduser(
GlobalPaths.MARO_LOCAL_TMP)
tar_script = f"tar czf {maro_local_tmp_abs_path}/{tmp_file_name} -C {folder_name} {basename}"
_ = SubProcess.run(tar_script)
copy_script = (
f"scp {maro_local_tmp_abs_path}/{tmp_file_name} "
f"{admin_username}@{node_ip_address}:{GlobalPaths.MARO_LOCAL_TMP}")
_ = SubProcess.run(copy_script)
untar_script = (
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"'tar xzf {GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name} -C {target_dir}'"
)
_ = SubProcess.run(untar_script)
remove_script = f"rm {maro_local_tmp_abs_path}/{tmp_file_name}"
_ = SubProcess.run(remove_script)
remote_remove_script = (
f"ssh -o StrictHostKeyChecking=no -p {ssh_port} {admin_username}@{node_ip_address} "
f"'rm {GlobalPaths.MARO_LOCAL_TMP}/{tmp_file_name}'")
_ = SubProcess.run(remote_remove_script)
def push_data(self, local_path: str, remote_dir: str):
# Load details
cluster_details = self.cluster_details
cluster_id = cluster_details['id']
# Get sas
sas = self._check_and_get_account_sas()
# Push data
source_path = get_reformatted_source_path(local_path)
target_dir = get_reformatted_target_dir(remote_dir)
if not target_dir.startswith("/"):
raise CliException("Invalid remote path")
copy_command = f'azcopy copy ' \
f'"{source_path}" ' \
f'"https://{cluster_id}st.file.core.windows.net/{cluster_id}-fs{target_dir}?{sas}" ' \
f'--recursive=True'
_ = SubProcess.run(copy_command)
def copy_files_from_node(local_dir: str, remote_path: str, admin_username: str, node_ip_address: str) -> None:
"""Copy node files to local, automatically create folder if not exist.
Args:
local_dir (str): dir for local files
remote_path (str): path of the remote file
admin_username (str)
node_ip_address (str)
"""
source_path = get_reformatted_source_path(remote_path)
basename = os.path.basename(source_path)
folder_name = os.path.dirname(source_path)
target_dir = get_reformatted_target_dir(local_dir)
mkdir_script = f"mkdir -p {target_dir}"
_ = SubProcess.run(mkdir_script)
copy_script = (f"ssh {admin_username}@{node_ip_address} 'tar czf - -C {folder_name} {basename}' | "
f"tar xzf - -C {target_dir}")
_ = SubProcess.run(copy_script)
def copy_files_to_node(local_path: str, remote_dir: str, admin_username: str, node_ip_address: str) -> None:
"""Copy local files to node, automatically create folder if not exist.
Args:
local_path (str): path of the local file
remote_dir (str): dir for remote files
admin_username (str)
node_ip_address (str)
"""
source_path = get_reformatted_source_path(local_path)
basename = os.path.basename(source_path)
folder_name = os.path.dirname(source_path)
target_dir = get_reformatted_target_dir(remote_dir)
mkdir_script = f"ssh -o StrictHostKeyChecking=no {admin_username}@{node_ip_address} 'mkdir -p {target_dir}'"
_ = SubProcess.run(mkdir_script)
copy_script = (f"tar czf - -C {folder_name} {basename} | "
f"ssh {admin_username}@{node_ip_address} 'tar xzf - -C {target_dir}'")
_ = SubProcess.run(copy_script)
def pull_data(self, local_dir: str, remote_path: str):
# Load details
cluster_details = self.cluster_details
cluster_id = cluster_details['id']
# Get sas
sas = self._check_and_get_account_sas()
# Push data
local_dir = os.path.expanduser(local_dir)
source_path = get_reformatted_source_path(remote_path)
target_dir = get_reformatted_target_dir(local_dir)
mkdir_script = f"mkdir -p {target_dir}"
_ = SubProcess.run(mkdir_script)
if not source_path.startswith("/"):
raise CliException("Invalid remote path")
copy_command = f'azcopy copy ' \
f'"https://{cluster_id}st.file.core.windows.net/{cluster_id}-fs{source_path}?{sas}" ' \
f'"{os.path.expanduser(target_dir)}" ' \
f'--recursive=True'
_ = SubProcess.run(copy_command)