def ssh_launch_json2mc(filepath, server, engine, labeling): """ simple method that uploads the file whose path is provided as argument filepath and remotely launches a verification task in background :param filepath: path of the .json which has to be uploaded on the server and provided as a parameter to json2mc.py """ d_vert_server_hostname = D_VERT_SERVER_HOSTNAME[server] base_json2mc_path = BASE_JSON2MC_PATH[server] username = D_VERT_SERVER_USER[server] exp_dir = EXP_DIR print('ssh_launch_json2mc({})'.format(filepath)) destination_path = os.path.join(base_json2mc_path, exp_dir, os.path.basename(filepath)) out_path = os.path.join(base_json2mc_path, exp_dir) # log_path = os.path.join(BASE_JSON2MC_PATH, 'logs', '{}.log'.format(os.path.splitext(os.path.basename(filepath))[0])) print('connecting to {}'.format(d_vert_server_hostname)) rem = ParamikoMachine(host=d_vert_server_hostname, keyfile=config.PRIVATE_KEY_PATH, user=username) print('uploading\n{}\nto\n{}:{}'.format(filepath, d_vert_server_hostname, destination_path)) mkdir = rem['mkdir'] mkdir['-p', os.path.join(base_json2mc_path, exp_dir)] rem.env.path.insert(0, [ '/home/fmbiased/DICE/Francesco/zot/bin:/home/fmbiased/DICE/Francesco/z3/bin:/home/fmbiased/uppaal64-4.1.19/bin-Linux' ]) rem.upload(filepath, destination_path) with rem.cwd(base_json2mc_path): activate_venv = rem['./activate_venv.sh'] run_json2mc = rem['./run_json2mc.py'] print('activating venv...') activate_venv() print('launching json2mc...') # run_json2mc['-T', 'spark', '--db', '-c', destination_path, '-o', out_path] & FG sleep(random.uniform(0, 3)) if labeling: # TODO improve this f = run_json2mc['-T', 'spark', '-e', engine, '--db', '-l', '-c', destination_path, '-o', out_path] & BG else: f = run_json2mc['-T', 'spark', '-e', engine, '--db', '-c', destination_path, '-o', out_path] & BG #f = (run_json2mc['-T', 'spark', '--db', '-c', destination_path] > 'pine.log')() & BG print('launched {}'.format(f)) sleep(1) f.wait() if f.ready(): print('Command exited with return_code {}\nSTDOUT:{}\nSTDERR:{}'. format(f.returncode, f.stdout, f.stderr)) else: print('Command running in background...\n{}'.format(f))
class SSHClient(ClosingContextManager): def __init__(self, host, username, password=None, private_key_path=None, passphrase_required=False, port=22): super().__init__() self._host = host self._username = username self._password = password self._private_key_path = private_key_path self._passphrase_required = passphrase_required self._port = port self._paramiko_machine = None @property def host(self): return self._host @property def sftp(self): return self._paramiko_machine.sftp def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy()) def __enter__(self): self.connect() return self def close(self): if self._paramiko_machine is not None: self._paramiko_machine.close() self._paramiko_machine = None def upload(self, localpath, remotepath="."): self._paramiko_machine.upload(localpath, remotepath) def download(self, remote_path, local_path): self._paramiko_machine.download(remote_path, local_path) def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False, expected_return_code=None, in_separate_shell=False): if in_background: command = "sh -c {}".format(shlex.quote(command + " &")) elif in_separate_shell: command = "sh -c {}".format(shlex.quote(command)) if sudo: command = "sudo " + command command = _preflight_command + " ; " + command with self._paramiko_machine.session(get_pty or sudo) as session: _, stdout, stderr = session.run(command, retcode=expected_return_code) return (stdout + " " + stderr).strip() def sudo_exec(self, command, *args, **kwargs): self.ssh_exec(command, *args, sudo=True, **kwargs) def check_output(self, command, expected_output, *args, **kwargs): matched_message = "Matched" execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format( command, shlex.quote(expected_output), matched_message) return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs) def path_exists(self, path): found_msg = "Path exists." check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg) return found_msg in self.ssh_exec(check) def ssh_copy_id(self, public_key_path): if not self.path_exists("~/.ssh"): self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh") if not self.path_exists("~/.ssh/authorized_keys"): self.ssh_exec("touch ~/.ssh/authorized_keys" " && chmod 600 ~/.ssh/authorized_keys") public_key = pathlib.Path(public_key_path).read_text() copy_key_command = ( """ KEY={};""" """ if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];""" """ then echo $KEY >> ~/.ssh/authorized_keys; fi""") self.ssh_exec(copy_key_command.format(shlex.quote(public_key))) def is_process_running(self, process_name): stdout = self.ssh_exec("pgrep -c " + process_name) return int(stdout) > 0 def is_java_running(self): return self.is_process_running("java") def check_file_contains_text(self, text_file, text_to_find): contains_message = "Contains" check_command = "if grep -q {} {} ; then echo '{}'; fi".format( shlex.quote(text_to_find), text_file, contains_message) return contains_message in self.ssh_exec(check_command) def has_ignite_started(self, log_file): if not self.is_java_running() or not self.path_exists(log_file): return False return self.check_file_contains_text(log_file, "Topology snapshot") def stop_process(self, process_name): if self.is_process_running(process_name): self.sudo_exec("killall -15 " + process_name) time.sleep(5) # Give process some time to finish. else: return False if self.is_process_running(process_name): self.sudo_exec("killall -9 " + process_name) return True def scp(self, remote_path, local_path="."): pkey_used = False # Purposefully do not try to circumvent passing passphrase to scp while using private key. if self._private_key_path is None or self._passphrase_required: scp_command = "scp -o StrictHostKeyChecking=no {} {}".format( local_path, remote_path) if self._password is not None: scp_command = ("sshpass -p {} " + scp_command).format( shlex.quote(self._password)) else: pkey_used = True pkey_file = os.path.basename(self._private_key_path) self.upload(self._private_key_path) self.ssh_exec("chmod 600 " + pkey_file) scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format( pkey_file, remote_path, local_path) downloaded = False retries = 0 while not downloaded and retries < 5: self.ssh_exec(scp_command) downloaded = self.path_exists(local_path) retries += 1 if pkey_used: self._paramiko_machine.sftp.remove(pkey_file) return downloaded def modify_file(self, path, load_hook, modify_hook, dump_hook): with self._paramiko_machine.sftp.open(path, mode="rU") as file: content = load_hook(file) modify_hook(content) # Paramiko's BufferedFile isn't seekable # so we should reopen it to overwrite. with self._paramiko_machine.sftp.open(path, mode="wU") as file: dump_hook(content, file) def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info( "Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host) def get_available_memory(self, unit=bitmath.Byte): free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'")) return unit(bytes=free_bytes) def wget(self, url): file_name = url.rsplit("/", 1)[1] if self.path_exists(file_name): return self.ssh_exec("wget " + url)
class SSHClient(ClosingContextManager): def __init__(self, host, username, password=None, private_key_path=None, passphrase_required=False, port=22): super().__init__() self._host = host self._username = username self._password = password self._private_key_path = private_key_path self._passphrase_required = passphrase_required self._port = port self._paramiko_machine = None @property def host(self): return self._host @property def sftp(self): return self._paramiko_machine.sftp def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy()) def __enter__(self): self.connect() return self def close(self): if self._paramiko_machine is not None: self._paramiko_machine.close() self._paramiko_machine = None def upload(self, localpath, remotepath="."): self._paramiko_machine.upload(localpath, remotepath) def download(self, remote_path, local_path): self._paramiko_machine.download(remote_path, local_path) def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False, expected_return_code=None, in_separate_shell=False): if in_background: command = "sh -c {}".format(shlex.quote(command + " &")) elif in_separate_shell: command = "sh -c {}".format(shlex.quote(command)) if sudo: command = "sudo " + command command = _preflight_command + " ; " + command with self._paramiko_machine.session(get_pty or sudo) as session: _, stdout, stderr = session.run(command, retcode=expected_return_code) return (stdout + " " + stderr).strip() def sudo_exec(self, command, *args, **kwargs): self.ssh_exec(command, *args, sudo=True, **kwargs) def check_output(self, command, expected_output, *args, **kwargs): matched_message = "Matched" execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format( command, shlex.quote(expected_output), matched_message) return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs) def path_exists(self, path): found_msg = "Path exists." check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg) return found_msg in self.ssh_exec(check) def ssh_copy_id(self, public_key_path): if not self.path_exists("~/.ssh"): self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh") if not self.path_exists("~/.ssh/authorized_keys"): self.ssh_exec("touch ~/.ssh/authorized_keys" " && chmod 600 ~/.ssh/authorized_keys") public_key = pathlib.Path(public_key_path).read_text() copy_key_command = (""" KEY={};""" """ if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];""" """ then echo $KEY >> ~/.ssh/authorized_keys; fi""") self.ssh_exec(copy_key_command.format(shlex.quote(public_key))) def is_process_running(self, process_name): stdout = self.ssh_exec("pgrep -c " + process_name) return int(stdout) > 0 def is_java_running(self): return self.is_process_running("java") def check_file_contains_text(self, text_file, text_to_find): contains_message = "Contains" check_command = "if grep -q {} {} ; then echo '{}'; fi".format( shlex.quote(text_to_find), text_file, contains_message) return contains_message in self.ssh_exec(check_command) def has_ignite_started(self, log_file): if not self.is_java_running() or not self.path_exists(log_file): return False return self.check_file_contains_text(log_file, "Topology snapshot") def stop_process(self, process_name): if self.is_process_running(process_name): self.sudo_exec("killall -15 " + process_name) time.sleep(5) # Give process some time to finish. else: return False if self.is_process_running(process_name): self.sudo_exec("killall -9 " + process_name) return True def scp(self, remote_path, local_path="."): pkey_used = False # Purposefully do not try to circumvent passing passphrase to scp while using private key. if self._private_key_path is None or self._passphrase_required: scp_command = "scp -o StrictHostKeyChecking=no {} {}".format(local_path, remote_path) if self._password is not None: scp_command = ("sshpass -p {} " + scp_command).format( shlex.quote(self._password)) else: pkey_used = True pkey_file = os.path.basename(self._private_key_path) self.upload(self._private_key_path) self.ssh_exec("chmod 600 " + pkey_file) scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format( pkey_file, remote_path, local_path) downloaded = False retries = 0 while not downloaded and retries < 5: self.ssh_exec(scp_command) downloaded = self.path_exists(local_path) retries += 1 if pkey_used: self._paramiko_machine.sftp.remove(pkey_file) return downloaded def modify_file(self, path, load_hook, modify_hook, dump_hook): with self._paramiko_machine.sftp.open(path, mode="rU") as file: content = load_hook(file) modify_hook(content) # Paramiko's BufferedFile isn't seekable # so we should reopen it to overwrite. with self._paramiko_machine.sftp.open(path, mode="wU") as file: dump_hook(content, file) def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info("Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host) def get_available_memory(self, unit=bitmath.Byte): free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'")) return unit(bytes=free_bytes) def wget(self, url): file_name = url.rsplit("/", 1)[1] if self.path_exists(file_name): return self.ssh_exec("wget " + url)