class RemoteConnection():
def __init__(self, host, user, password=None, port=22):
"""
Initialize connection to a remote host
:param host: hostname to connect to
:param user: username to connect as
:param password: the password of the user
:param port: the port number on the host machine
"""
self.host, self.user, self.port = host, user, port
self.remote = ParamikoMachine(
self.host,
user=self.user,
password=password,
port=self.port,
missing_host_policy=paramiko.AutoAddPolicy())
def list_remote_files(self, remote_dir, prefix):
"""
List files in folder on a remote host which start with a given prefix
:param remote_dir: the absolute location of the folder to search
:param prefix: the beginning of all files names to find
:return: list of remote full paths
"""
r_ls = self.remote['ls']
files = r_ls(remote_dir).split('\n')
if len(files):
files = [
os.path.join(remote_dir, f) for f in files
if f.startswith(prefix)
]
return files
else:
raise ValueError('No files found at {host}:{loc}'.format(
host=self.host, loc=remote_dir))
def download_remote_file(self, remote_path, local_path):
"""
Transfer a file from a remote host locally
:param remote_path: the absolute location of the file to grab (including host)
:param local_path: the local file to write to
"""
self.remote.download(remote_path, local_path)
class SSHClient(ClosingContextManager):
def __init__(self,
host,
username,
password=None,
private_key_path=None,
passphrase_required=False,
port=22):
super().__init__()
self._host = host
self._username = username
self._password = password
self._private_key_path = private_key_path
self._passphrase_required = passphrase_required
self._port = port
self._paramiko_machine = None
@property
def host(self):
return self._host
@property
def sftp(self):
return self._paramiko_machine.sftp
def connect(self, reconnect=False, timeout=5):
if self._paramiko_machine is not None:
if not reconnect:
return
self.close()
if self._paramiko_machine is None:
self._paramiko_machine = ParamikoMachine(
host=self._host,
password=self._password,
user=self._username,
connect_timeout=timeout,
keyfile=self._private_key_path,
missing_host_policy=paramiko.AutoAddPolicy())
def __enter__(self):
self.connect()
return self
def close(self):
if self._paramiko_machine is not None:
self._paramiko_machine.close()
self._paramiko_machine = None
def upload(self, localpath, remotepath="."):
self._paramiko_machine.upload(localpath, remotepath)
def download(self, remote_path, local_path):
self._paramiko_machine.download(remote_path, local_path)
def ssh_exec(self,
command,
sudo=False,
in_background=False,
get_pty=False,
expected_return_code=None,
in_separate_shell=False):
if in_background:
command = "sh -c {}".format(shlex.quote(command + " &"))
elif in_separate_shell:
command = "sh -c {}".format(shlex.quote(command))
if sudo:
command = "sudo " + command
command = _preflight_command + " ; " + command
with self._paramiko_machine.session(get_pty or sudo) as session:
_, stdout, stderr = session.run(command,
retcode=expected_return_code)
return (stdout + " " + stderr).strip()
def sudo_exec(self, command, *args, **kwargs):
self.ssh_exec(command, *args, sudo=True, **kwargs)
def check_output(self, command, expected_output, *args, **kwargs):
matched_message = "Matched"
execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format(
command, shlex.quote(expected_output), matched_message)
return matched_message in self.ssh_exec(execute_and_check_command,
*args, **kwargs)
def path_exists(self, path):
found_msg = "Path exists."
check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg)
return found_msg in self.ssh_exec(check)
def ssh_copy_id(self, public_key_path):
if not self.path_exists("~/.ssh"):
self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh")
if not self.path_exists("~/.ssh/authorized_keys"):
self.ssh_exec("touch ~/.ssh/authorized_keys"
" && chmod 600 ~/.ssh/authorized_keys")
public_key = pathlib.Path(public_key_path).read_text()
copy_key_command = (
""" KEY={};"""
""" if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];"""
""" then echo $KEY >> ~/.ssh/authorized_keys; fi""")
self.ssh_exec(copy_key_command.format(shlex.quote(public_key)))
def is_process_running(self, process_name):
stdout = self.ssh_exec("pgrep -c " + process_name)
return int(stdout) > 0
def is_java_running(self):
return self.is_process_running("java")
def check_file_contains_text(self, text_file, text_to_find):
contains_message = "Contains"
check_command = "if grep -q {} {} ; then echo '{}'; fi".format(
shlex.quote(text_to_find), text_file, contains_message)
return contains_message in self.ssh_exec(check_command)
def has_ignite_started(self, log_file):
if not self.is_java_running() or not self.path_exists(log_file):
return False
return self.check_file_contains_text(log_file, "Topology snapshot")
def stop_process(self, process_name):
if self.is_process_running(process_name):
self.sudo_exec("killall -15 " + process_name)
time.sleep(5) # Give process some time to finish.
else:
return False
if self.is_process_running(process_name):
self.sudo_exec("killall -9 " + process_name)
return True
def scp(self, remote_path, local_path="."):
pkey_used = False
# Purposefully do not try to circumvent passing passphrase to scp while using private key.
if self._private_key_path is None or self._passphrase_required:
scp_command = "scp -o StrictHostKeyChecking=no {} {}".format(
local_path, remote_path)
if self._password is not None:
scp_command = ("sshpass -p {} " + scp_command).format(
shlex.quote(self._password))
else:
pkey_used = True
pkey_file = os.path.basename(self._private_key_path)
self.upload(self._private_key_path)
self.ssh_exec("chmod 600 " + pkey_file)
scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format(
pkey_file, remote_path, local_path)
downloaded = False
retries = 0
while not downloaded and retries < 5:
self.ssh_exec(scp_command)
downloaded = self.path_exists(local_path)
retries += 1
if pkey_used:
self._paramiko_machine.sftp.remove(pkey_file)
return downloaded
def modify_file(self, path, load_hook, modify_hook, dump_hook):
with self._paramiko_machine.sftp.open(path, mode="rU") as file:
content = load_hook(file)
modify_hook(content)
# Paramiko's BufferedFile isn't seekable
# so we should reopen it to overwrite.
with self._paramiko_machine.sftp.open(path, mode="wU") as file:
dump_hook(content, file)
def reboot(self, timeout=60, reconnect_attempts=5):
command_schedule_time = 10
self.sudo_exec("shutdown -r -t {}".format(command_schedule_time))
self.close()
time.sleep(command_schedule_time)
retries = 0
for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts):
try:
self.connect()
return
except _reconnect_exceptions:
retries += 1
logger.info(
"Trying to connect to %s after reboot issued for the %d time.",
self._host, retries)
raise RuntimeError("Unable to connect to %s after issuing reboot.",
self._host)
def get_available_memory(self, unit=bitmath.Byte):
free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'"))
return unit(bytes=free_bytes)
def wget(self, url):
file_name = url.rsplit("/", 1)[1]
if self.path_exists(file_name):
return
self.ssh_exec("wget " + url)
class SSHClient(ClosingContextManager):
def __init__(self, host, username, password=None, private_key_path=None,
passphrase_required=False, port=22):
super().__init__()
self._host = host
self._username = username
self._password = password
self._private_key_path = private_key_path
self._passphrase_required = passphrase_required
self._port = port
self._paramiko_machine = None
@property
def host(self):
return self._host
@property
def sftp(self):
return self._paramiko_machine.sftp
def connect(self, reconnect=False, timeout=5):
if self._paramiko_machine is not None:
if not reconnect:
return
self.close()
if self._paramiko_machine is None:
self._paramiko_machine = ParamikoMachine(
host=self._host, password=self._password, user=self._username, connect_timeout=timeout,
keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy())
def __enter__(self):
self.connect()
return self
def close(self):
if self._paramiko_machine is not None:
self._paramiko_machine.close()
self._paramiko_machine = None
def upload(self, localpath, remotepath="."):
self._paramiko_machine.upload(localpath, remotepath)
def download(self, remote_path, local_path):
self._paramiko_machine.download(remote_path, local_path)
def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False,
expected_return_code=None, in_separate_shell=False):
if in_background:
command = "sh -c {}".format(shlex.quote(command + " &"))
elif in_separate_shell:
command = "sh -c {}".format(shlex.quote(command))
if sudo:
command = "sudo " + command
command = _preflight_command + " ; " + command
with self._paramiko_machine.session(get_pty or sudo) as session:
_, stdout, stderr = session.run(command, retcode=expected_return_code)
return (stdout + " " + stderr).strip()
def sudo_exec(self, command, *args, **kwargs):
self.ssh_exec(command, *args, sudo=True, **kwargs)
def check_output(self, command, expected_output, *args, **kwargs):
matched_message = "Matched"
execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format(
command, shlex.quote(expected_output), matched_message)
return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs)
def path_exists(self, path):
found_msg = "Path exists."
check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg)
return found_msg in self.ssh_exec(check)
def ssh_copy_id(self, public_key_path):
if not self.path_exists("~/.ssh"):
self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh")
if not self.path_exists("~/.ssh/authorized_keys"):
self.ssh_exec("touch ~/.ssh/authorized_keys"
" && chmod 600 ~/.ssh/authorized_keys")
public_key = pathlib.Path(public_key_path).read_text()
copy_key_command = (""" KEY={};"""
""" if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];"""
""" then echo $KEY >> ~/.ssh/authorized_keys; fi""")
self.ssh_exec(copy_key_command.format(shlex.quote(public_key)))
def is_process_running(self, process_name):
stdout = self.ssh_exec("pgrep -c " + process_name)
return int(stdout) > 0
def is_java_running(self):
return self.is_process_running("java")
def check_file_contains_text(self, text_file, text_to_find):
contains_message = "Contains"
check_command = "if grep -q {} {} ; then echo '{}'; fi".format(
shlex.quote(text_to_find), text_file, contains_message)
return contains_message in self.ssh_exec(check_command)
def has_ignite_started(self, log_file):
if not self.is_java_running() or not self.path_exists(log_file):
return False
return self.check_file_contains_text(log_file, "Topology snapshot")
def stop_process(self, process_name):
if self.is_process_running(process_name):
self.sudo_exec("killall -15 " + process_name)
time.sleep(5) # Give process some time to finish.
else:
return False
if self.is_process_running(process_name):
self.sudo_exec("killall -9 " + process_name)
return True
def scp(self, remote_path, local_path="."):
pkey_used = False
# Purposefully do not try to circumvent passing passphrase to scp while using private key.
if self._private_key_path is None or self._passphrase_required:
scp_command = "scp -o StrictHostKeyChecking=no {} {}".format(local_path, remote_path)
if self._password is not None:
scp_command = ("sshpass -p {} " + scp_command).format(
shlex.quote(self._password))
else:
pkey_used = True
pkey_file = os.path.basename(self._private_key_path)
self.upload(self._private_key_path)
self.ssh_exec("chmod 600 " + pkey_file)
scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format(
pkey_file, remote_path, local_path)
downloaded = False
retries = 0
while not downloaded and retries < 5:
self.ssh_exec(scp_command)
downloaded = self.path_exists(local_path)
retries += 1
if pkey_used:
self._paramiko_machine.sftp.remove(pkey_file)
return downloaded
def modify_file(self, path, load_hook, modify_hook, dump_hook):
with self._paramiko_machine.sftp.open(path, mode="rU") as file:
content = load_hook(file)
modify_hook(content)
# Paramiko's BufferedFile isn't seekable
# so we should reopen it to overwrite.
with self._paramiko_machine.sftp.open(path, mode="wU") as file:
dump_hook(content, file)
def reboot(self, timeout=60, reconnect_attempts=5):
command_schedule_time = 10
self.sudo_exec("shutdown -r -t {}".format(command_schedule_time))
self.close()
time.sleep(command_schedule_time)
retries = 0
for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts):
try:
self.connect()
return
except _reconnect_exceptions:
retries += 1
logger.info("Trying to connect to %s after reboot issued for the %d time.", self._host, retries)
raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host)
def get_available_memory(self, unit=bitmath.Byte):
free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'"))
return unit(bytes=free_bytes)
def wget(self, url):
file_name = url.rsplit("/", 1)[1]
if self.path_exists(file_name):
return
self.ssh_exec("wget " + url)