def ssh_check_output(
client: paramiko.client.SSHClient,
command: str,
timeout_seconds: int=None,
):
"""
Run a command via the provided SSH client and return the output captured
on stdout.
Raise an exception if the command returns a non-zero code.
"""
stdin, stdout, stderr = client.exec_command(
command,
get_pty=True,
timeout=timeout_seconds)
# NOTE: Paramiko doesn't clearly document this, but we must read() before
# calling recv_exit_status().
# See: https://github.com/paramiko/paramiko/issues/448#issuecomment-159481997
stdout_output = stdout.read().decode('utf8').rstrip('\n')
stderr_output = stderr.read().decode('utf8').rstrip('\n')
exit_status = stdout.channel.recv_exit_status()
if exit_status:
# TODO: Return a custom exception that includes the return code.
# See: https://docs.python.org/3/library/subprocess.html#subprocess.check_output
# NOTE: We are losing the output order here since output from stdout and stderr
# may be interleaved.
raise SSHError(
host=client.get_transport().getpeername()[0],
message=stdout_output + stderr_output)
return stdout_output
def tdloadViaSSH(engine: sqlalchemy.engine.base.Engine,
sshc: paramiko.client.SSHClient,
tablename: str,
targetPath: str,
jobname: str,
dbname: str = None,
skipRowNum: int = 0,
verbose: bool = True) -> None:
targetPath = pathlib.Path(targetPath)
if dbname is None:
dbname = engine.url.database
# always use option: QuotedData = 'Optional'
options = "--DCPQuotedData 'Optional'"
if skipRowNum > 0:
options += f" --SourceSkipRows {skipRowNum}"
tdload_command = (f"tdload -f {targetPath} -t {dbname}.{tablename}" +
f" -h {engine.url.host} -u {engine.url.username}" +
f" -p {engine.url.password}" +
f" --TargetWorkingDatabase {dbname}" +
f" {options} {jobname}")
# drop error log table if exists
dropIfExists(tablename + "_ET", dbname, engine)
dropIfExists(tablename + "_UV", dbname, engine)
# execute command via ssh
stdin, stdout, stderr = sshc.exec_command(tdload_command)
for line in stdout:
if verbose:
print(line)
else:
if re.match(r".*(Total Rows|successfully).*", line):
print(line)
def run_experiment(
ssh_client: paramiko.client.SSHClient,
experiment_name: str,
duration: int,
exposure_time: float = None,
) -> ExperimentStreams:
"""Run run_experiment (image capture program) on the cosmobot with the given name and duration
Args:
experiment_name: experiment name to pass to run_experiment
duration: duration to pass to run_experiment
Returns: ExperimentStreams object
"""
run_experiment_command = _generate_run_experiment_command(
experiment_name, duration, exposure_time
)
hostname = ssh_client.get_transport().hostname
logger.info(
f"Starting image capture on cosmobot {hostname}\n"
f"Command: {run_experiment_command}"
)
return ExperimentStreams(*ssh_client.exec_command(run_experiment_command))
def ssh_check_output(client: paramiko.client.SSHClient, command: str):
"""
Run a command via the provided SSH client and return the output captured
on stdout.
Raise an exception if the command returns a non-zero code.
"""
stdin, stdout, stderr = client.exec_command(command, get_pty=True)
# NOTE: Paramiko doesn't clearly document this, but we must read() before
# calling recv_exit_status().
# See: https://github.com/paramiko/paramiko/issues/448#issuecomment-159481997
stdout_output = stdout.read().decode('utf8').rstrip('\n')
stderr_output = stderr.read().decode('utf8').rstrip('\n')
exit_status = stdout.channel.recv_exit_status()
if exit_status:
# TODO: Return a custom exception that includes the return code.
# See: https://docs.python.org/3/library/subprocess.html#subprocess.check_output
# NOTE: We are losing the output order here since output from stdout and stderr
# may be interleaved.
raise SSHError(host=client.get_transport().getpeername()[0],
message=stdout_output + stderr_output)
return stdout_output
def unarchiveSSH(archivePath: pathlib.PosixPath,
sshc: paramiko.client.SSHClient,
unarchiveFolder: pathlib.PosixPath = None,
method: str = "gz",
verbose: bool = True) -> None:
if unarchiveFolder is None:
unarchiveFolder = archivePath.parent
if method == "7z":
command = f"7z e {archivePath} -o{unarchiveFolder}"
elif method in ["gz", "xz", "bz2"]:
d_format_option = {"gz": "z", "xz": "J", "bz2": "j"}
command = (f"tar -xv{d_format_option[method]}f" +
f"{archivePath} -C {unarchiveFolder}")
else:
raise ValueError(f"method only supports ['7z', 'gz', 'xz', 'bz2']")
with verbosity_context(f"Unarchiving {archivePath}", verbose):
stdin, stdout, stderr = sshc.exec_command(command)
if verbose:
for line in stdout:
print(line)
def download_checkpoint(ssh_client: paramiko.client.SSHClient,
sftp_client: paramiko.sftp_client.SFTPClient,
run_id: str) -> None:
remote_checkpoint_dir = f'/home/{user}/proj/Wave-U-Net/checkpoints/{run_id}/'
_stdin, stdout, _stderr = ssh_client.exec_command(
f'ls -1 -v {remote_checkpoint_dir} -I checkpoint | tail -n 1')
last_checkpoint_filename = stdout.read().decode('utf8')
if not last_checkpoint_filename:
print('File not found')
return
last_checkpoint_name = os.path.splitext(last_checkpoint_filename)[0]
checkpoint_files = [
last_checkpoint_name + '.index', last_checkpoint_name + '.meta',
last_checkpoint_name + '.data-00000-of-00001'
]
local_folder = os.path.join(output_dir, run_id)
os.makedirs(local_folder, exist_ok=True)
for filename in checkpoint_files:
local_path = os.path.join(local_folder, filename)
print(os.path.join(remote_checkpoint_dir, filename))
sftp_client.get(os.path.join(remote_checkpoint_dir, filename),
local_path)
