def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy())
def ssh_launch_json2mc(filepath, server, engine, labeling): """ simple method that uploads the file whose path is provided as argument filepath and remotely launches a verification task in background :param filepath: path of the .json which has to be uploaded on the server and provided as a parameter to json2mc.py """ d_vert_server_hostname = D_VERT_SERVER_HOSTNAME[server] base_json2mc_path = BASE_JSON2MC_PATH[server] username = D_VERT_SERVER_USER[server] exp_dir = EXP_DIR print('ssh_launch_json2mc({})'.format(filepath)) destination_path = os.path.join(base_json2mc_path, exp_dir, os.path.basename(filepath)) out_path = os.path.join(base_json2mc_path, exp_dir) # log_path = os.path.join(BASE_JSON2MC_PATH, 'logs', '{}.log'.format(os.path.splitext(os.path.basename(filepath))[0])) print('connecting to {}'.format(d_vert_server_hostname)) rem = ParamikoMachine(host=d_vert_server_hostname, keyfile=config.PRIVATE_KEY_PATH, user=username) print('uploading\n{}\nto\n{}:{}'.format(filepath, d_vert_server_hostname, destination_path)) mkdir = rem['mkdir'] mkdir['-p', os.path.join(base_json2mc_path, exp_dir)] rem.env.path.insert(0, [ '/home/fmbiased/DICE/Francesco/zot/bin:/home/fmbiased/DICE/Francesco/z3/bin:/home/fmbiased/uppaal64-4.1.19/bin-Linux' ]) rem.upload(filepath, destination_path) with rem.cwd(base_json2mc_path): activate_venv = rem['./activate_venv.sh'] run_json2mc = rem['./run_json2mc.py'] print('activating venv...') activate_venv() print('launching json2mc...') # run_json2mc['-T', 'spark', '--db', '-c', destination_path, '-o', out_path] & FG sleep(random.uniform(0, 3)) if labeling: # TODO improve this f = run_json2mc['-T', 'spark', '-e', engine, '--db', '-l', '-c', destination_path, '-o', out_path] & BG else: f = run_json2mc['-T', 'spark', '-e', engine, '--db', '-c', destination_path, '-o', out_path] & BG #f = (run_json2mc['-T', 'spark', '--db', '-c', destination_path] > 'pine.log')() & BG print('launched {}'.format(f)) sleep(1) f.wait() if f.ready(): print('Command exited with return_code {}\nSTDOUT:{}\nSTDERR:{}'. format(f.returncode, f.stdout, f.stderr)) else: print('Command running in background...\n{}'.format(f))
def __init__(self, host, user, password=None, port=22): """ Initialize connection to a remote host :param host: hostname to connect to :param user: username to connect as :param password: the password of the user :param port: the port number on the host machine """ self.host, self.user, self.port = host, user, port self.remote = ParamikoMachine( self.host, user=self.user, password=password, port=self.port, missing_host_policy=paramiko.AutoAddPolicy())
def test_deploy_paramiko(self): rem = ParamikoMachine("localhost", missing_host_policy=paramiko.AutoAddPolicy()) with DeployedServer(rem) as dep: conn = dep.classic_connect() print(conn.modules.sys) func = conn.modules.os.getcwd print(func()) try: func() except EOFError: pass else: self.fail("expected an EOFError") rem.close()
def _get_machine(host, user, keyfile): try: for i in range(10): logger.debug(f'Connection attempt no: {i}') try: signal.alarm(15) ret = ParamikoMachine(host=host, user=user, keyfile=keyfile, connect_timeout=15, keep_alive=30, missing_host_policy=AutoAddPolicy()) signal.alarm(0) except TimeoutError: pass else: return ret else: raise RuntimeError('Connection could not be established') finally: signal.alarm(0)
class RemoteConnection(): def __init__(self, host, user, password=None, port=22): """ Initialize connection to a remote host :param host: hostname to connect to :param user: username to connect as :param password: the password of the user :param port: the port number on the host machine """ self.host, self.user, self.port = host, user, port self.remote = ParamikoMachine( self.host, user=self.user, password=password, port=self.port, missing_host_policy=paramiko.AutoAddPolicy()) def list_remote_files(self, remote_dir, prefix): """ List files in folder on a remote host which start with a given prefix :param remote_dir: the absolute location of the folder to search :param prefix: the beginning of all files names to find :return: list of remote full paths """ r_ls = self.remote['ls'] files = r_ls(remote_dir).split('\n') if len(files): files = [ os.path.join(remote_dir, f) for f in files if f.startswith(prefix) ] return files else: raise ValueError('No files found at {host}:{loc}'.format( host=self.host, loc=remote_dir)) def download_remote_file(self, remote_path, local_path): """ Transfer a file from a remote host locally :param remote_path: the absolute location of the file to grab (including host) :param local_path: the local file to write to """ self.remote.download(remote_path, local_path)
def _build_remote(self, host): (hostname, plumbum_kwargs, forward_agent) = load_ssh_config_for_plumbum("~/.ssh/config", host) print("[ssh] host={host} hostname={hostname} other_conf={other_conf}". format(host=host, hostname=hostname, other_conf=plumbum_kwargs)) remote = ParamikoMachine(hostname, password=None, **plumbum_kwargs) if forward_agent == "yes": print("[ssh] forwarding local agent") enable_agent_forwarding(remote._client) return remote
def pb_remote(self): """ plumbum ParamikoMachine remote machine Returns: `plumbum.machines.paramiko_machine.ParamikoMachine`: """ if self._pb_remote: # and self._pb_remote._session.alive(): if self._pb_remote._session.alive(): return self._pb_remote self._pb_remote = ParamikoMachine( self.ssh_host, user='******', port=self.ssh_port, keyfile=self.client._get_ssh_key_file(), missing_host_policy=AutoAddPolicy) return self._pb_remote
def _build_remote(self, host): (hostname, plumbum_kwargs, forward_agent) = self._load_ssh_config_for_plumbum( '~/.ssh/config', host) print('[ssh] host={host} hostname={hostname} other_conf={other_conf}'. format( host=host, hostname=hostname, other_conf=plumbum_kwargs, )) remote = ParamikoMachine(hostname, password=None, **plumbum_kwargs) if forward_agent == 'yes': print('[ssh] forwarding local agent') self._forward_local_agent(remote._client) return remote
def test_deploy_paramiko(self): try: import paramiko # @UnusedImport except Exception: self.skipTest("Paramiko is not available") from plumbum.machines.paramiko_machine import ParamikoMachine rem = ParamikoMachine("localhost", missing_host_policy = paramiko.AutoAddPolicy()) with DeployedServer(rem) as dep: conn = dep.classic_connect() print (conn.modules.sys) func = conn.modules.os.getcwd print (func()) try: func() except EOFError: pass else: self.fail("expected an EOFError")
def get_paramiko_ssh(location, **kwargs): from paramiko import AutoAddPolicy from plumbum.machines.paramiko_machine import ParamikoMachine password = location.facts.get('password') keyfile = location.facts.get('keyfile') settings = { 'user': location.facts.get('user', 'root'), 'port': location.facts.get('port', 22), 'look_for_keys': False, 'missing_host_policy': AutoAddPolicy(), 'keep_alive': 60 } if password: settings['password'] = password if keyfile: settings['keyfile'] = keyfile ssh = ParamikoMachine(location.hostname, **settings) ssh.sftp.walk = types.MethodType(walk, ssh.sftp) return ssh
def go(): creds = getCreds() print("Connecting...") mach = ParamikoMachine(creds['remote address'], user=creds['remote user'], keyfile=creds['remote key'], missing_host_policy=ignoreIt()) print("Deploying RPC interface...") server = DeployedServer(mach) print("Connected.") conn = server.classic_connect() import webFunctions webFunctions.urllib.request = conn.modules['urllib.request'] wg = webFunctions.WebGetRobust() print(wg.getpage('http://api.ipify.org?format=json')) import ScrapePlugins.SadPandaLoader.Run ScrapePlugins.SadPandaLoader.Run.test()
logger = logging.getLogger('deploy') if __name__ == '__main__': host = local.env['DEPLOY_HOST'] user = local.env['DEPLOY_USER'] remote_dir = local.env['DEPLOY_DIR'] deploy_key = local.env['DEPLOY_KEY'] deploy_key_file = BASE_DIR / 'tmp' deploy_key_file.write_text('\n'.join(deploy_key.split('|'))) try: with ParamikoMachine( host, user=user, keyfile=str(deploy_key_file), load_system_host_keys=False, missing_host_policy=paramiko.AutoAddPolicy(), ) as rem: rem.upload(BASE_DIR / 'deploy' / 'docker-compose.yml', f'{remote_dir}/docker-compose.yml') logger.info('Copy %s', 'docker-compose.yml') rem.upload(BASE_DIR / 'deploy' / 'nginx.conf', f'{remote_dir}/nginx.conf') logger.info('Copy %s', 'nginx.conf') docker_compose = rem['docker-compose'] docker_compose['down', '--rmi', 'all']() logger.info('docker-compose stop') docker_compose['up', '-d']() logger.info('docker-compose up')
def _connect(self): return ParamikoMachine(TEST_HOST, missing_host_policy = paramiko.AutoAddPolicy())
def connect(instance_id): instance = ec2.Instance(instance_id) return ParamikoMachine(instance.public_ip_address, user="******", keyfile=settings.KEY_FILE, missing_host_policy=paramiko.AutoAddPolicy())
def measure(self): t0=pc() frecuencia_min = np.log10(self.sd.def_cfg['f_inicial']['value']) frecuencia_max= np.log10(self.sd.def_cfg['f_final']['value']) puntos_decada= self.sd.def_cfg['n_puntos']['value'] ampl =self.sd.def_cfg['vosc']['value'] decimation=1*puntos_decada*[8192]+2*puntos_decada*[1024]+1*puntos_decada*[64]+2*puntos_decada*[1] numero_valores=int((frecuencia_max-frecuencia_min)*puntos_decada) if (self.sd.def_cfg['tipo_barrido']['value']==0): self.sd.freq = np.linspace(self.sd.def_cfg['f_inicial']['value'], self.sd.def_cfg['f_final']['value'], self.sd.def_cfg['n_puntos']['value']) elif(self.sd.def_cfg['tipo_barrido']['value']==1): self.sd.freq = np.logspace(np.log10(self.sd.def_cfg['f_inicial']['value']), np.log10(self.sd.def_cfg['f_final']['value']), numero_valores,base=10) # valores no configurables desde el front-end wave_form = 'sine' Rs=1000 fm=125000000 numero_pulsos=10 ciclos=5 R_shunt_k = self.sd.def_cfg['shunt']['value'] #elijo 1000 Postprocesamiento=self.sd.def_cfg['postprocesamiento']['value'] # print(R_shunt_k) # # borra cuando quite la SOURCE 2 # amplreference=np.linspace(10,1,numero_valores) # ampl2=1/amplreference # phases=np.linspace(180,0,numero_valores) shunt=[10.0,100.0,1000.0,10000.0,100000.0,1300000.0] #configuro via i2c la resistencia de shunt # veamos = SshMachine(self.host, user = "******") veamos = ParamikoMachine(self.host, user = "******", password="******") veamos.env["LD_LIBRARY_PATH"]="/opt/redpitaya/lib" veamos.cwd.chdir("/opt/redpitaya/bin") comando="./i2c_shunt " + str(R_shunt_k) # print (comando) r_back = veamos[comando] # sizeh1=str('-sizeh1={0}'.format(individual[0])) # sizeh2=str('-sizeh2={0}'.format(individual[1])) # epochs=str('-epochs={0}'.format(epochs)) # decay=str('-decay={0}'.format(decay_value)) # step=str('-step={0}'.format(learning_step)) # minibatch=str('-batchsize={0}'.format(batchsize)) # idea=str(r_back[epochs, sizeh1,sizeh2, minibatch,decay,step]()) # for line in idea.split("\n"): # if "error_val" in line: # #print (line.strip()) # error_rate_float=[float(s) for s in re.findall('\d+\.\d+',line)] # error_rate=error_rate_float[0] veamos.close() self.dv.append_plus("Midiendo Z=R+iX") iteracion=1 configuracion=0 adquisicion=0 postprocesamiento=0 espera_trigger=0 Z=np.zeros(numero_valores) PHASE=np.zeros(numero_valores) for freq in (self.sd.freq): t1=pc() self.tx_txt('GEN:RST') self.tx_txt('ACQ:RST') self.tx_txt('ACQ:DATA:UNITS VOLTS') self.tx_txt('SOUR1:BURS:STAT BURST') # % Set burst mode to ON self.tx_txt('SOUR1:BURS:NCYC 10') # Set 10 pulses of sine wave self.tx_txt('SOUR1:FUNC ' + str(wave_form).upper()) self.tx_txt('SOUR1:VOLT ' + str(ampl)) self.tx_txt('SOUR1:FREQ:FIX ' + str(freq)) #rp_s.tx_txt('SOUR1:TRIG:SOUR INT') #rp_s.tx_txt('SOUR1:TRIG:IMM') # esto habrá que borrarlo !! # self.tx_txt('SOUR2:FUNC ' + str(wave_form).upper()) # self.tx_txt('SOUR2:VOLT ' + str(ampl2[iteracion-1])) # self.tx_txt('SOUR2:FREQ:FIX ' + str(freq)) # self.tx_txt('SOUR2:BURS:STAT BURST') # % Set burst mode to ON # self.tx_txt('SOUR2:BURS:NCYC 10') # Set 10 pulses of sine wave # self.tx_txt('SOUR2:PHAS ' + str(phases[iteracion-1])) # hay que borrarlo self.tx_txt('ACQ:DEC ' + str(decimation[iteracion-1])) self.tx_txt('ACQ:START') self.tx_txt('ACQ:TRIG CH1_PE') self.tx_txt('OUTPUT:STATE ON') #cuidado con cambiar esto t2=pc() # ESPERAMOS EL TRIGGER # sleep(1) while freq<100: self.tx_txt('ACQ:TRIG:STAT?') if self.rx_txt() == 'TD': break t2t=pc() # self.dv.append_plus('iteracion:'+ str(iteracion)) # self.dv.append_plus('frecuencia:'+ str(freq)) print (iteracion) print(freq) sleep(0.2) self.tx_txt('ACQ:TPOS?') veamos= self.rx_txt() posicion_trigger=int(veamos) # self.dv.append_plus("posicion trigger:" + str(posicion_trigger)) print('posicion trigger:',veamos) length=fm*(ciclos)/(freq*decimation[iteracion-1]) # print('decimation:',decimation[iteracion-1]) # LEEMOS Y REPRESENTAMOS 1 self.tx_txt('ACQ:SOUR1:DATA:STA:N? ' + str(posicion_trigger)+','+ str(length)) t3=pc() buff_string = self.rx_txt() buff_string = buff_string.strip('{}\n\r').replace(" ", "").split(',') buff = list(map(float, buff_string)) t4=pc() my_array = np.asarray(buff) # super_buffer.append(buff) # super_buffer_flat=sum(super_buffer, []) t5=pc() # LEEMOS Y REPRESENTAMOS2 self.tx_txt('ACQ:SOUR2:DATA:STA:N? ' + str(posicion_trigger)+','+ str(length)) t6=pc() buff_string = self.rx_txt() buff_string = buff_string.strip('{}\n\r').replace(" ", "").split(',') buff = list(map(float, buff_string)) t7=pc() my_array2 = np.asarray(buff) dif=my_array-my_array2 #super_buffer2.append(buff) #super_buffer_flat2=sum(super_buffer2, []) if postprocesamiento==0: yf = fft(my_array) yf2 = fft(my_array2) yf3=fft(dif) indice1=np.argmax(np.abs(yf)) indice2=np.argmax(np.abs(yf2)) indice3=np.argmax(np.abs(yf3)) # Zq=np.max(np.abs(yf)) # Zr=np.max(np.abs(yf2)) # Z[iteracion-1]=np.max(np.abs(yf))/np.max(np.abs(yf2)) # primera opcion #yf_dif=(yf2-yf) # indice3=np.argmax(np.abs(yf_dif)) ## Z[iteracion-1]=np.max(np.abs(yf_dif))*1000/np.max(np.abs(yf2)) # segunda opcion Z[iteracion-1]=np.max(np.abs(yf3))*shunt[R_shunt_k]/np.max(np.abs(yf2)) #tercera opcion PHASE[iteracion-1]=((np.angle(yf2[indice2]/yf[indice1])))*180/np.pi else : # metodo alternativo T=(1/125e6)*decimation[iteracion-1]; # Sampling time [seconds] # time increment w_out=freq*2*np.pi U_dut=dif I_dut=my_array2/shunt[R_shunt_k] N=int(length) U_dut_sampled_X=np.zeros(N) U_dut_sampled_Y=np.zeros(N) I_dut_sampled_X=np.zeros(N) I_dut_sampled_Y=np.zeros(N) dT=np.zeros(N) # Accurie signals U_in_1 and U_in_2 from RedPitaya for N-sampels time and Calculate for Lock in and save in vectors for sampels for t in range(N): U_dut_sampled_X[t]=U_dut[t]*np.sin(t*T*w_out) U_dut_sampled_Y[t]=U_dut[t]*np.sin(t*T*w_out-np.pi/2) I_dut_sampled_X[t]=I_dut[t]*np.sin(t*T*w_out) I_dut_sampled_Y[t]=I_dut[t]*np.sin(t*T*w_out-np.pi/2) dT[t]=t*T # Calculate two components of lock-in for both mesured voltage signals X_component_lock_in_1=np.trapz(U_dut_sampled_X,x=dT) Y_component_lock_in_1=np.trapz(U_dut_sampled_Y,dT) X_component_lock_in_2=np.trapz(I_dut_sampled_X,dT) Y_component_lock_in_2=np.trapz(I_dut_sampled_Y,dT) U_dut_amp=(np.sqrt((X_component_lock_in_1)**2+(Y_component_lock_in_1)**2))*2 Phase_U_dut=np.arctan2(Y_component_lock_in_1,X_component_lock_in_1) # Calculate amplitude and angle of I_dut I_dut_amp=(np.sqrt((X_component_lock_in_2)**2+(Y_component_lock_in_2)**2))*2 Phase_I_dut=np.arctan2(Y_component_lock_in_2,X_component_lock_in_2) # Calculate amplitude of current trough impedance and amplitude of impedance Z_amp=(U_dut_amp/I_dut_amp); # Amplitude of impedance Z[iteracion-1]=Z_amp Phase_Z_rad=(Phase_U_dut-Phase_I_dut) Phase_check=(Phase_U_dut-Phase_I_dut)*(180/np.pi) if (Phase_check<=-180): Phase_Z=(Phase_U_dut-Phase_I_dut)*(180/np.pi)+360 else: if (Phase_check>=180): Phase_Z=(Phase_U_dut-Phase_I_dut)*(180/np.pi)-360 else: Phase_Z=Phase_check PHASE[iteracion-1]=Phase_Z t8=pc() # N2=my_array2.shape[0] # T=1/fm # idea=N//2 # xf2 = fftfreq(N2, T)[:N2//2] # plot2=plt.figure(2*(iteracion-1)+2) # plt.plot(xf2, 2.0/N * np.abs(yf2[0:N2//2])) # plt.plot(xf2, p[0:N2//2]) # plt.xlabel('Frecuencia') # plt.grid() iteracion=iteracion+1 self.tx_txt('OUTPUT:STATE OFF') self.tx_txt('ACQ:STOP') t9=pc() postprocesamiento=postprocesamiento+(t5-t4)+(t8-t7) configuracion=configuracion+(t2-t1)+(t9-t8)+(t6-t5)+(t3-t2t) adquisicion=adquisicion+(t4-t3)+(t7-t6) espera_trigger=espera_trigger+ (t2t-t2) # # Service Request instead of using pulling # event_type = visa.constants.EventType.service_request # # Mechanism by which we want to be notified # event_mech = visa.constants.EventMechanism.queue # self.inst.write('TRGS INT') # Internal Trigger Source # self.inst.write('ESNB 1') # Event_Status_Register[0]=1 // Enables Sweep Completion bit # self.inst.write('*SRE 4') # Service Request Enable = 1 # self.inst.write('*CLS') # Clears Error queue # self.inst.write('MEAS IRIM') # Medida de R y X # self.inst.write('HIDI OFF') # Muestras la traza inactiva # self.inst.write('SING') # Iniciar un barrido único. # self.dv.append_plus("ACK Instrumento = %s" % self.inst.query('*OPC?')) # self.inst.enable_event(event_type, event_mech) # # #self.dv.append_plus(self.inst.query('*OPC?')) # # Wait for the event to occur # response = self.inst.wait_on_event(event_type, 10000) # # #if (response.event.event_type == event_type): # # response.timed_out = False # self.inst.disable_event(event_type, event_mech) # response.timed_out = False # Recover Measured Data # self.inst.write('TRAC A') # Selecciona traza A # self.inst.write('AUTO') # Autoescala # aux_R = np.fromstring(self.inst.query('OUTPDTRC?'), dtype=float, sep=',') # self.inst.write('TRAC B') # Selecciona traza A # self.inst.write('AUTO') # Autoescala # aux_X = np.fromstring(self.inst.query('OUTPDTRC?'), dtype=float, sep=',') t10=pc() self.sd.R_data = Z*np.cos(PHASE*np.pi/180) self.sd.X_data = Z*np.sin(PHASE*np.pi/180) # Compute Err, Eri, Er_mod, Er_fase_data # First create frequency array based on actual gui conditions # The freq array will not be changed until next data acquisition even if GUI changes complex_aux = self.sd.R_data + self.sd.X_data*1j self.sd.Z_mod_data = Z self.sd.Z_fase_data = PHASE admitance_aux = 1./complex_aux G_data = np.real(admitance_aux) Cp_data = np.imag(admitance_aux)/(2*np.pi*self.sd.freq) self.sd.Err_data = Cp_data/self.sd.Co self.sd.Eri_data = G_data/(self.sd.Co*(2*np.pi*self.sd.freq)); E_data = self.sd.Err_data + -1*self.sd.Eri_data*1j; self.sd.Er_mod_data = np.abs(E_data); self.sd.Er_fase_data = np.angle(E_data); t11=pc() # # Deactivate BIAS for security reasons # self.inst.write('DCO OFF') # self.inst.write('DCRNG M1') postprocesamiento=postprocesamiento+(t11-t10) print('espera_trigger:',espera_trigger) print('configuracion:',configuracion) print('adquisicion:',adquisicion) print('postprocesamiento:',postprocesamiento) total=t11-t0 print ('total:',total) absolute_val_array = np.abs(self.sd.freq - 1000) smallest_difference_index = absolute_val_array.argmin() print ('R_data =', self.sd.R_data[smallest_difference_index]) print ('X_data=', self.sd.X_data[smallest_difference_index]) print ('resistencia shunt=', shunt[R_shunt_k]) # self.inst.wait_for_srq(self.sd.def_cfg['GPIB_timeout']) self.dv.append_plus("He finalizado de medir") self.dv.append_plus("tiempo transcurrido:" + str(total))
class SSHClient(ClosingContextManager): def __init__(self, host, username, password=None, private_key_path=None, passphrase_required=False, port=22): super().__init__() self._host = host self._username = username self._password = password self._private_key_path = private_key_path self._passphrase_required = passphrase_required self._port = port self._paramiko_machine = None @property def host(self): return self._host @property def sftp(self): return self._paramiko_machine.sftp def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy()) def __enter__(self): self.connect() return self def close(self): if self._paramiko_machine is not None: self._paramiko_machine.close() self._paramiko_machine = None def upload(self, localpath, remotepath="."): self._paramiko_machine.upload(localpath, remotepath) def download(self, remote_path, local_path): self._paramiko_machine.download(remote_path, local_path) def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False, expected_return_code=None, in_separate_shell=False): if in_background: command = "sh -c {}".format(shlex.quote(command + " &")) elif in_separate_shell: command = "sh -c {}".format(shlex.quote(command)) if sudo: command = "sudo " + command command = _preflight_command + " ; " + command with self._paramiko_machine.session(get_pty or sudo) as session: _, stdout, stderr = session.run(command, retcode=expected_return_code) return (stdout + " " + stderr).strip() def sudo_exec(self, command, *args, **kwargs): self.ssh_exec(command, *args, sudo=True, **kwargs) def check_output(self, command, expected_output, *args, **kwargs): matched_message = "Matched" execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format( command, shlex.quote(expected_output), matched_message) return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs) def path_exists(self, path): found_msg = "Path exists." check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg) return found_msg in self.ssh_exec(check) def ssh_copy_id(self, public_key_path): if not self.path_exists("~/.ssh"): self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh") if not self.path_exists("~/.ssh/authorized_keys"): self.ssh_exec("touch ~/.ssh/authorized_keys" " && chmod 600 ~/.ssh/authorized_keys") public_key = pathlib.Path(public_key_path).read_text() copy_key_command = ( """ KEY={};""" """ if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];""" """ then echo $KEY >> ~/.ssh/authorized_keys; fi""") self.ssh_exec(copy_key_command.format(shlex.quote(public_key))) def is_process_running(self, process_name): stdout = self.ssh_exec("pgrep -c " + process_name) return int(stdout) > 0 def is_java_running(self): return self.is_process_running("java") def check_file_contains_text(self, text_file, text_to_find): contains_message = "Contains" check_command = "if grep -q {} {} ; then echo '{}'; fi".format( shlex.quote(text_to_find), text_file, contains_message) return contains_message in self.ssh_exec(check_command) def has_ignite_started(self, log_file): if not self.is_java_running() or not self.path_exists(log_file): return False return self.check_file_contains_text(log_file, "Topology snapshot") def stop_process(self, process_name): if self.is_process_running(process_name): self.sudo_exec("killall -15 " + process_name) time.sleep(5) # Give process some time to finish. else: return False if self.is_process_running(process_name): self.sudo_exec("killall -9 " + process_name) return True def scp(self, remote_path, local_path="."): pkey_used = False # Purposefully do not try to circumvent passing passphrase to scp while using private key. if self._private_key_path is None or self._passphrase_required: scp_command = "scp -o StrictHostKeyChecking=no {} {}".format( local_path, remote_path) if self._password is not None: scp_command = ("sshpass -p {} " + scp_command).format( shlex.quote(self._password)) else: pkey_used = True pkey_file = os.path.basename(self._private_key_path) self.upload(self._private_key_path) self.ssh_exec("chmod 600 " + pkey_file) scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format( pkey_file, remote_path, local_path) downloaded = False retries = 0 while not downloaded and retries < 5: self.ssh_exec(scp_command) downloaded = self.path_exists(local_path) retries += 1 if pkey_used: self._paramiko_machine.sftp.remove(pkey_file) return downloaded def modify_file(self, path, load_hook, modify_hook, dump_hook): with self._paramiko_machine.sftp.open(path, mode="rU") as file: content = load_hook(file) modify_hook(content) # Paramiko's BufferedFile isn't seekable # so we should reopen it to overwrite. with self._paramiko_machine.sftp.open(path, mode="wU") as file: dump_hook(content, file) def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info( "Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host) def get_available_memory(self, unit=bitmath.Byte): free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'")) return unit(bytes=free_bytes) def wget(self, url): file_name = url.rsplit("/", 1)[1] if self.path_exists(file_name): return self.ssh_exec("wget " + url)
def _connect(self): if paramiko is None: pytest.skip("System does not have paramiko installed") return ParamikoMachine(TEST_HOST, missing_host_policy=paramiko.AutoAddPolicy())
class SSHClient(ClosingContextManager): def __init__(self, host, username, password=None, private_key_path=None, passphrase_required=False, port=22): super().__init__() self._host = host self._username = username self._password = password self._private_key_path = private_key_path self._passphrase_required = passphrase_required self._port = port self._paramiko_machine = None @property def host(self): return self._host @property def sftp(self): return self._paramiko_machine.sftp def connect(self, reconnect=False, timeout=5): if self._paramiko_machine is not None: if not reconnect: return self.close() if self._paramiko_machine is None: self._paramiko_machine = ParamikoMachine( host=self._host, password=self._password, user=self._username, connect_timeout=timeout, keyfile=self._private_key_path, missing_host_policy=paramiko.AutoAddPolicy()) def __enter__(self): self.connect() return self def close(self): if self._paramiko_machine is not None: self._paramiko_machine.close() self._paramiko_machine = None def upload(self, localpath, remotepath="."): self._paramiko_machine.upload(localpath, remotepath) def download(self, remote_path, local_path): self._paramiko_machine.download(remote_path, local_path) def ssh_exec(self, command, sudo=False, in_background=False, get_pty=False, expected_return_code=None, in_separate_shell=False): if in_background: command = "sh -c {}".format(shlex.quote(command + " &")) elif in_separate_shell: command = "sh -c {}".format(shlex.quote(command)) if sudo: command = "sudo " + command command = _preflight_command + " ; " + command with self._paramiko_machine.session(get_pty or sudo) as session: _, stdout, stderr = session.run(command, retcode=expected_return_code) return (stdout + " " + stderr).strip() def sudo_exec(self, command, *args, **kwargs): self.ssh_exec(command, *args, sudo=True, **kwargs) def check_output(self, command, expected_output, *args, **kwargs): matched_message = "Matched" execute_and_check_command = "if {} | grep -q {} ; then echo '{}'; fi".format( command, shlex.quote(expected_output), matched_message) return matched_message in self.ssh_exec(execute_and_check_command, *args, **kwargs) def path_exists(self, path): found_msg = "Path exists." check = "if [ -e {} ]; then echo '{}'; fi".format(path, found_msg) return found_msg in self.ssh_exec(check) def ssh_copy_id(self, public_key_path): if not self.path_exists("~/.ssh"): self.ssh_exec("mkdir ~/.ssh && chmod 700 ~/.ssh") if not self.path_exists("~/.ssh/authorized_keys"): self.ssh_exec("touch ~/.ssh/authorized_keys" " && chmod 600 ~/.ssh/authorized_keys") public_key = pathlib.Path(public_key_path).read_text() copy_key_command = (""" KEY={};""" """ if [ -z "$(grep "$KEY" ~/.ssh/authorized_keys )" ];""" """ then echo $KEY >> ~/.ssh/authorized_keys; fi""") self.ssh_exec(copy_key_command.format(shlex.quote(public_key))) def is_process_running(self, process_name): stdout = self.ssh_exec("pgrep -c " + process_name) return int(stdout) > 0 def is_java_running(self): return self.is_process_running("java") def check_file_contains_text(self, text_file, text_to_find): contains_message = "Contains" check_command = "if grep -q {} {} ; then echo '{}'; fi".format( shlex.quote(text_to_find), text_file, contains_message) return contains_message in self.ssh_exec(check_command) def has_ignite_started(self, log_file): if not self.is_java_running() or not self.path_exists(log_file): return False return self.check_file_contains_text(log_file, "Topology snapshot") def stop_process(self, process_name): if self.is_process_running(process_name): self.sudo_exec("killall -15 " + process_name) time.sleep(5) # Give process some time to finish. else: return False if self.is_process_running(process_name): self.sudo_exec("killall -9 " + process_name) return True def scp(self, remote_path, local_path="."): pkey_used = False # Purposefully do not try to circumvent passing passphrase to scp while using private key. if self._private_key_path is None or self._passphrase_required: scp_command = "scp -o StrictHostKeyChecking=no {} {}".format(local_path, remote_path) if self._password is not None: scp_command = ("sshpass -p {} " + scp_command).format( shlex.quote(self._password)) else: pkey_used = True pkey_file = os.path.basename(self._private_key_path) self.upload(self._private_key_path) self.ssh_exec("chmod 600 " + pkey_file) scp_command = "scp -i {} -o StrictHostKeyChecking=no {} {}".format( pkey_file, remote_path, local_path) downloaded = False retries = 0 while not downloaded and retries < 5: self.ssh_exec(scp_command) downloaded = self.path_exists(local_path) retries += 1 if pkey_used: self._paramiko_machine.sftp.remove(pkey_file) return downloaded def modify_file(self, path, load_hook, modify_hook, dump_hook): with self._paramiko_machine.sftp.open(path, mode="rU") as file: content = load_hook(file) modify_hook(content) # Paramiko's BufferedFile isn't seekable # so we should reopen it to overwrite. with self._paramiko_machine.sftp.open(path, mode="wU") as file: dump_hook(content, file) def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info("Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host) def get_available_memory(self, unit=bitmath.Byte): free_bytes = int(self.ssh_exec("free -b | gawk '/Mem:/{print $7}'")) return unit(bytes=free_bytes) def wget(self, url): file_name = url.rsplit("/", 1)[1] if self.path_exists(file_name): return self.ssh_exec("wget " + url)