class RemoteConnection(): def __init__(self, host, user, password=None, port=22): """ Initialize connection to a remote host :param host: hostname to connect to :param user: username to connect as :param password: the password of the user :param port: the port number on the host machine """ self.host, self.user, self.port = host, user, port self.remote = ParamikoMachine( self.host, user=self.user, password=password, port=self.port, missing_host_policy=paramiko.AutoAddPolicy()) def list_remote_files(self, remote_dir, prefix): """ List files in folder on a remote host which start with a given prefix :param remote_dir: the absolute location of the folder to search :param prefix: the beginning of all files names to find :return: list of remote full paths """ r_ls = self.remote['ls'] files = r_ls(remote_dir).split('\n') if len(files): files = [ os.path.join(remote_dir, f) for f in files if f.startswith(prefix) ] return files else: raise ValueError('No files found at {host}:{loc}'.format( host=self.host, loc=remote_dir)) def download_remote_file(self, remote_path, local_path): """ Transfer a file from a remote host locally :param remote_path: the absolute location of the file to grab (including host) :param local_path: the local file to write to """ self.remote.download(remote_path, local_path)
class SSHClient(ClosingContextManager): def __init__(self, host, username, password=None, private_key_path=None, passphrase_required=False, port=22): super().__init__() self._host = host self._username = username self._password = password self._private_key_path = private_key_path self._passphrase_required = passphrase_required self._port = port self._paramiko_machine = None @property def host(self): return self._host @property def sftp(self): return self._paramiko_machine.sftp def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy()) def __enter__(self): self.connect() return self def close(self): if self._paramiko_machine is not None: self._paramiko_machine.close() self._paramiko_machine = None def upload(self, localpath, remotepath="."): self._paramiko_machine.upload(localpath, remotepath) def download(self, remote_path, local_path): self._paramiko_machine.download(remote_path, local_path) def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False, expected_return_code=None, in_separate_shell=False): if in_background: command = "sh -c {}".format(shlex.quote(command + " &")) elif in_separate_shell: command = "sh -c {}".format(shlex.quote(command)) if sudo: command = "sudo " + command command = _preflight_command + " ; " + command with self._paramiko_machine.session(get_pty or sudo) as session: _, stdout, stderr = session.run(command, retcode=expected_return_code) return (stdout + " " + stderr).strip() def sudo_exec(self, command, *args, **kwargs): self.ssh_exec(command, *args, sudo=True, **kwargs) def check_output(self, command, expected_output, *args, **kwargs): matched_message = "Matched" execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format( command, shlex.quote(expected_output), matched_message) return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs) def path_exists(self, path): found_msg = "Path exists." check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg) return found_msg in self.ssh_exec(check) def ssh_copy_id(self, public_key_path): if not self.path_exists("~/.ssh"): self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh") if not self.path_exists("~/.ssh/authorized_keys"): self.ssh_exec("touch ~/.ssh/authorized_keys" " && chmod 600 ~/.ssh/authorized_keys") public_key = pathlib.Path(public_key_path).read_text() copy_key_command = ( """ KEY={};""" """ if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];""" """ then echo $KEY >> ~/.ssh/authorized_keys; fi""") self.ssh_exec(copy_key_command.format(shlex.quote(public_key))) def is_process_running(self, process_name): stdout = self.ssh_exec("pgrep -c " + process_name) return int(stdout) > 0 def is_java_running(self): return self.is_process_running("java") def check_file_contains_text(self, text_file, text_to_find): contains_message = "Contains" check_command = "if grep -q {} {} ; then echo '{}'; fi".format( shlex.quote(text_to_find), text_file, contains_message) return contains_message in self.ssh_exec(check_command) def has_ignite_started(self, log_file): if not self.is_java_running() or not self.path_exists(log_file): return False return self.check_file_contains_text(log_file, "Topology snapshot") def stop_process(self, process_name): if self.is_process_running(process_name): self.sudo_exec("killall -15 " + process_name) time.sleep(5) # Give process some time to finish. else: return False if self.is_process_running(process_name): self.sudo_exec("killall -9 " + process_name) return True def scp(self, remote_path, local_path="."): pkey_used = False # Purposefully do not try to circumvent passing passphrase to scp while using private key. if self._private_key_path is None or self._passphrase_required: scp_command = "scp -o StrictHostKeyChecking=no {} {}".format( local_path, remote_path) if self._password is not None: scp_command = ("sshpass -p {} " + scp_command).format( shlex.quote(self._password)) else: pkey_used = True pkey_file = os.path.basename(self._private_key_path) self.upload(self._private_key_path) self.ssh_exec("chmod 600 " + pkey_file) scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format( pkey_file, remote_path, local_path) downloaded = False retries = 0 while not downloaded and retries < 5: self.ssh_exec(scp_command) downloaded = self.path_exists(local_path) retries += 1 if pkey_used: self._paramiko_machine.sftp.remove(pkey_file) return downloaded def modify_file(self, path, load_hook, modify_hook, dump_hook): with self._paramiko_machine.sftp.open(path, mode="rU") as file: content = load_hook(file) modify_hook(content) # Paramiko's BufferedFile isn't seekable # so we should reopen it to overwrite. with self._paramiko_machine.sftp.open(path, mode="wU") as file: dump_hook(content, file) def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info( "Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host) def get_available_memory(self, unit=bitmath.Byte): free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'")) return unit(bytes=free_bytes) def wget(self, url): file_name = url.rsplit("/", 1)[1] if self.path_exists(file_name): return self.ssh_exec("wget " + url)
class SSHClient(ClosingContextManager): def __init__(self, host, username, password=None, private_key_path=None, passphrase_required=False, port=22): super().__init__() self._host = host self._username = username self._password = password self._private_key_path = private_key_path self._passphrase_required = passphrase_required self._port = port self._paramiko_machine = None @property def host(self): return self._host @property def sftp(self): return self._paramiko_machine.sftp def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy()) def __enter__(self): self.connect() return self def close(self): if self._paramiko_machine is not None: self._paramiko_machine.close() self._paramiko_machine = None def upload(self, localpath, remotepath="."): self._paramiko_machine.upload(localpath, remotepath) def download(self, remote_path, local_path): self._paramiko_machine.download(remote_path, local_path) def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False, expected_return_code=None, in_separate_shell=False): if in_background: command = "sh -c {}".format(shlex.quote(command + " &")) elif in_separate_shell: command = "sh -c {}".format(shlex.quote(command)) if sudo: command = "sudo " + command command = _preflight_command + " ; " + command with self._paramiko_machine.session(get_pty or sudo) as session: _, stdout, stderr = session.run(command, retcode=expected_return_code) return (stdout + " " + stderr).strip() def sudo_exec(self, command, *args, **kwargs): self.ssh_exec(command, *args, sudo=True, **kwargs) def check_output(self, command, expected_output, *args, **kwargs): matched_message = "Matched" execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format( command, shlex.quote(expected_output), matched_message) return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs) def path_exists(self, path): found_msg = "Path exists." check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg) return found_msg in self.ssh_exec(check) def ssh_copy_id(self, public_key_path): if not self.path_exists("~/.ssh"): self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh") if not self.path_exists("~/.ssh/authorized_keys"): self.ssh_exec("touch ~/.ssh/authorized_keys" " && chmod 600 ~/.ssh/authorized_keys") public_key = pathlib.Path(public_key_path).read_text() copy_key_command = (""" KEY={};""" """ if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];""" """ then echo $KEY >> ~/.ssh/authorized_keys; fi""") self.ssh_exec(copy_key_command.format(shlex.quote(public_key))) def is_process_running(self, process_name): stdout = self.ssh_exec("pgrep -c " + process_name) return int(stdout) > 0 def is_java_running(self): return self.is_process_running("java") def check_file_contains_text(self, text_file, text_to_find): contains_message = "Contains" check_command = "if grep -q {} {} ; then echo '{}'; fi".format( shlex.quote(text_to_find), text_file, contains_message) return contains_message in self.ssh_exec(check_command) def has_ignite_started(self, log_file): if not self.is_java_running() or not self.path_exists(log_file): return False return self.check_file_contains_text(log_file, "Topology snapshot") def stop_process(self, process_name): if self.is_process_running(process_name): self.sudo_exec("killall -15 " + process_name) time.sleep(5) # Give process some time to finish. else: return False if self.is_process_running(process_name): self.sudo_exec("killall -9 " + process_name) return True def scp(self, remote_path, local_path="."): pkey_used = False # Purposefully do not try to circumvent passing passphrase to scp while using private key. if self._private_key_path is None or self._passphrase_required: scp_command = "scp -o StrictHostKeyChecking=no {} {}".format(local_path, remote_path) if self._password is not None: scp_command = ("sshpass -p {} " + scp_command).format( shlex.quote(self._password)) else: pkey_used = True pkey_file = os.path.basename(self._private_key_path) self.upload(self._private_key_path) self.ssh_exec("chmod 600 " + pkey_file) scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format( pkey_file, remote_path, local_path) downloaded = False retries = 0 while not downloaded and retries < 5: self.ssh_exec(scp_command) downloaded = self.path_exists(local_path) retries += 1 if pkey_used: self._paramiko_machine.sftp.remove(pkey_file) return downloaded def modify_file(self, path, load_hook, modify_hook, dump_hook): with self._paramiko_machine.sftp.open(path, mode="rU") as file: content = load_hook(file) modify_hook(content) # Paramiko's BufferedFile isn't seekable # so we should reopen it to overwrite. with self._paramiko_machine.sftp.open(path, mode="wU") as file: dump_hook(content, file) def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info("Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host) def get_available_memory(self, unit=bitmath.Byte): free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'")) return unit(bytes=free_bytes) def wget(self, url): file_name = url.rsplit("/", 1)[1] if self.path_exists(file_name): return self.ssh_exec("wget " + url)