def ssh(self, command): """ Run a command in the OpenStack instance of the teuthology cluster. Return the stdout / stderr of the command. """ instance_id = self.get_instance_id(self.args.name) ip = self.get_floating_ip_or_ip(instance_id) client_args = { 'user_at_host': '@'.join((self.username, ip)), 'retry': False, } if self.key_filename: log.debug("ssh overriding key with " + self.key_filename) client_args['key_filename'] = self.key_filename client = connection.connect(**client_args) stdin, stdout, stderr = client.exec_command(command) stdout.channel.settimeout(300) out = '' try: out = stdout.read() log.debug('teardown stdout ' + command + ' ' + out) except Exception: log.exception('teardown ' + command + ' failed') err = stderr.read() log.debug('teardown stderr ' + command + ' ' + err) return out + ' ' + err
def ssh(self, command): """ Run a command in the OpenStack instance of the teuthology cluster. Return the stdout / stderr of the command. """ ip = self.instance.get_floating_ip_or_ip() client_args = { 'user_at_host': '@'.join((self.username, ip)), 'retry': False, 'timeout': 240, } if self.key_filename: log.debug("ssh overriding key with " + self.key_filename) client_args['key_filename'] = self.key_filename client = connection.connect(**client_args) # get the I/O channel to iterate line by line transport = client.get_transport() channel = transport.open_session() channel.get_pty() channel.settimeout(900) output = channel.makefile('r', 1) log.debug(":ssh@" + ip + ":" + command) channel.exec_command(command) for line in iter(output.readline, b''): log.info(line.strip()) return channel.recv_exit_status()
def test_connect_override_hostkeys(self): self.clear_config() m_ssh_instance = self.m_ssh.return_value = Mock() m_transport = Mock() m_ssh_instance.get_transport.return_value = m_transport m_host_keys = Mock() m_ssh_instance.get_host_keys.return_value = m_host_keys m_create_key = Mock() m_create_key.return_value = "frobnitz" got = connection.connect( '*****@*****.**', host_key='ssh-rsa testkey', _SSHClient=self.m_ssh, _create_key=m_create_key, ) self.m_ssh.assert_called_once() m_ssh_instance.get_host_keys.assert_called_once() m_host_keys.add.assert_called_once_with( hostname='orchestra.test.newdream.net.invalid', keytype='ssh-rsa', key='frobnitz', ) m_create_key.assert_called_once_with('ssh-rsa', 'testkey') m_ssh_instance.connect.assert_called_once_with( hostname='orchestra.test.newdream.net.invalid', username='******', timeout=60, ) m_transport.set_keepalive.assert_called_once_with(False) assert got is m_ssh_instance
def ssh(self, command): """ Run a command in the OpenStack instance of the teuthology cluster. Return the stdout / stderr of the command. """ instance_id = self.get_instance_id(self.args.name) ip = self.get_floating_ip_or_ip(instance_id) client_args = { 'user_at_host': '@'.join((self.username, ip)), 'retry': False, 'timeout': 240, } if self.key_filename: log.debug("ssh overriding key with " + self.key_filename) client_args['key_filename'] = self.key_filename client = connection.connect(**client_args) stdin, stdout, stderr = client.exec_command(command) stdout.channel.settimeout(300) out = '' try: out = stdout.read() log.debug('ssh stdout ' + command + ' ' + out) except Exception: log.exception('ssh ' + command + ' failed') err = stderr.read() log.debug('ssh stderr ' + command + ' ' + err) return out + ' ' + err
def test_connect_override_hostkeys(self): self.clear_config() fudge.clear_expectations() sshclient = fudge.Fake('SSHClient') ssh = sshclient.expects_call().with_args().returns_fake() ssh.remember_order() host_keys = fudge.Fake('HostKeys') host_keys.expects('add').with_args( hostname='orchestra.test.newdream.net.invalid', keytype='ssh-rsa', key='frobnitz', ) ssh.expects('get_host_keys').with_args().returns(host_keys) ssh.expects('connect').with_args( hostname='orchestra.test.newdream.net.invalid', username='******', timeout=60, ) transport = ssh.expects('get_transport').with_args().returns_fake() transport.remember_order() transport.expects('set_keepalive').with_args(False) create_key = fudge.Fake('create_key') create_key.expects_call().with_args('ssh-rsa', 'testkey').returns('frobnitz') got = connection.connect( '*****@*****.**', host_key='ssh-rsa testkey', _SSHClient=sshclient, _create_key=create_key, ) assert got is ssh
def test_and(self): ssh = connection.connect(HOST) r = run.run( client=ssh, args=['true', run.Raw('&&'), 'echo', 'yup'], stdout=StringIO(), ) assert r.stdout.getvalue() == 'yup\n'
def test_crash(self): ssh = connection.connect(HOST) e = assert_raises( CommandCrashedError, run.run, client=ssh, args=['sh', '-c', 'kill -ABRT $$'], ) assert e.command == "sh -c 'kill -ABRT $$'" assert str(e) == "Command crashed: \"sh -c 'kill -ABRT $$'\""
def test_lost(self): ssh = connection.connect(HOST) e = assert_raises( ConnectionLostError, run.run, client=ssh, args=['sh', '-c', 'kill -ABRT $PPID'], ) assert e.command == "sh -c 'kill -ABRT $PPID'" assert str(e) == \ "SSH connection was lost: \"sh -c 'kill -ABRT $PPID'\""
def test_pipe(self): ssh = connection.connect(HOST) r = run.run(client=ssh, args=["cat"], stdin=run.PIPE, stdout=StringIO(), wait=False) assert r.stdout.getvalue() == "" r.stdin.write("foo\n") r.stdin.write("bar\n") r.stdin.close() r.wait() got = r.exitstatus assert got == 0 assert r.stdout.getvalue() == "foo\nbar\n"
def test_lost(self): ssh = connection.connect(HOST) e = assert_raises( ConnectionLostError, run.run, client=ssh, args=['sh', '-c', 'kill -ABRT $PPID'], name=HOST, ) assert e.command == "sh -c 'kill -ABRT $PPID'" assert str(e) == \ "SSH connection to {host} was lost: ".format(host=HOST) + \ "\"sh -c 'kill -ABRT $PPID'\""
def connect(self, timeout=None, create_key=None, context='connect'): args = dict(user_at_host=self.name, host_key=self._host_key, keep_alive=self.keep_alive, _create_key=create_key) if context == 'reconnect': # The reason for the 'context' workaround is not very # clear from the technical side. # I'll get "[Errno 98] Address already in use" altough # there are no open tcp(ssh) connections. # When connecting without keepalive, host_key and _create_key # set, it will proceed. args = dict(user_at_host=self.name, _create_key=False, host_key=None) if timeout: args['timeout'] = timeout self.ssh = connection.connect(**args) return self.ssh
def test_pipe(self): ssh = connection.connect(HOST) r = run.run( client=ssh, args=['cat'], stdin=run.PIPE, stdout=StringIO(), wait=False, ) assert r.stdout.getvalue() == '' r.stdin.write('foo\n') r.stdin.write('bar\n') r.stdin.close() r.wait() got = r.exitstatus assert got == 0 assert r.stdout.getvalue() == 'foo\nbar\n'
def test_connect_no_verify_host_keys(self): self.clear_config() config.config.verify_host_keys = False fudge.clear_expectations() ssh = fudge.Fake('SSHClient') ssh.expects_call().with_args().returns(ssh) ssh.expects('set_missing_host_key_policy') ssh.expects('connect').with_args( hostname='orchestra.test.newdream.net.invalid', username='******', timeout=60, ) transport = ssh.expects('get_transport').with_args().returns_fake() transport.remember_order() transport.expects('set_keepalive').with_args(False) got = connection.connect( '*****@*****.**', _SSHClient=ssh, ) assert got is ssh
def test_connect_no_verify_host_keys(self): self.clear_config() config.config.verify_host_keys = False m_ssh_instance = self.m_ssh.return_value = Mock() m_transport = Mock() m_ssh_instance.get_transport.return_value = m_transport got = connection.connect( '*****@*****.**', _SSHClient=self.m_ssh, ) self.m_ssh.assert_called_once() m_ssh_instance.set_missing_host_key_policy.assert_called_once() assert not m_ssh_instance.load_system_host_keys.called m_ssh_instance.connect.assert_called_once_with( hostname='orchestra.test.newdream.net.invalid', username='******', timeout=60, ) m_transport.set_keepalive.assert_called_once_with(False) assert got is m_ssh_instance
def cloud_init_wait(self, instance): """ Wait for cloud-init to complete on the name_or_ip OpenStack instance. """ ip = instance.get_floating_ip_or_ip() log.debug('cloud_init_wait ' + ip) client_args = { 'user_at_host': '@'.join((self.username, ip)), 'timeout': 240, 'retry': False, } if self.key_filename: log.debug("using key " + self.key_filename) client_args['key_filename'] = self.key_filename with safe_while(sleep=30, tries=30, action="cloud_init_wait " + ip) as proceed: success = False # CentOS 6.6 logs in /var/log/clout-init-output.log # CentOS 7.0 logs in /var/log/clout-init.log tail = ("tail --follow=name --retry" " /var/log/cloud-init*.log /tmp/init.out") while proceed(): try: client = connection.connect(**client_args) except paramiko.PasswordRequiredException as e: raise Exception( "The private key requires a passphrase.\n" "Create a new key with:" " openstack keypair create myself > myself.pem\n" " chmod 600 myself.pem\n" "and call teuthology-openstack with the options\n" " --key-name myself --key-filename myself.pem\n") except paramiko.AuthenticationException as e: log.debug('cloud_init_wait AuthenticationException ' + str(e)) continue except socket.timeout as e: log.debug('cloud_init_wait connect socket.timeout ' + str(e)) continue except socket.error as e: log.debug('cloud_init_wait connect socket.error ' + str(e)) continue except Exception as e: transients = ('Incompatible ssh peer', 'Unknown server') for transient in transients: if transient in str(e): continue log.exception('cloud_init_wait ' + ip) raise log.debug('cloud_init_wait ' + tail) try: # get the I/O channel to iterate line by line transport = client.get_transport() channel = transport.open_session() channel.get_pty() channel.settimeout(240) output = channel.makefile('r', 1) channel.exec_command(tail) for line in iter(output.readline, b''): log.info(line.strip()) if self.up_string in line: success = True break except socket.timeout as e: client.close() log.debug('cloud_init_wait socket.timeout ' + tail) continue except socket.error as e: client.close() log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + tail) continue client.close() if success: break return success
def cloud_init_wait(self, name_or_ip): """ Wait for cloud-init to complete on the name_or_ip OpenStack instance. """ log.debug('cloud_init_wait ' + name_or_ip) client_args = { 'user_at_host': '@'.join((self.username, name_or_ip)), 'timeout': 240, 'retry': False, } if self.key_filename: log.debug("using key " + self.key_filename) client_args['key_filename'] = self.key_filename with safe_while(sleep=30, tries=100, action="cloud_init_wait " + name_or_ip) as proceed: success = False # CentOS 6.6 logs in /var/log/clout-init-output.log # CentOS 7.0 logs in /var/log/clout-init.log all_done = ("tail /var/log/cloud-init*.log ; " + " test -f /tmp/init.out && tail /tmp/init.out ; " + " grep '" + self.up_string + "' " + "/var/log/cloud-init*.log") while proceed(): try: client = connection.connect(**client_args) except paramiko.PasswordRequiredException as e: raise Exception( "The private key requires a passphrase.\n" "Create a new key with:" " openstack keypair create myself > myself.pem\n" " chmod 600 myself.pem\n" "and call teuthology-openstack with the options\n" " --key-name myself --key-filename myself.pem\n") except paramiko.AuthenticationException as e: log.debug('cloud_init_wait AuthenticationException ' + str(e)) continue except socket.timeout as e: log.debug('cloud_init_wait connect socket.timeout ' + str(e)) continue except socket.error as e: log.debug('cloud_init_wait connect socket.error ' + str(e)) continue except Exception as e: transients = ('Incompatible ssh peer', 'Unknown server') for transient in transients: if transient in str(e): continue log.exception('cloud_init_wait ' + name_or_ip) raise log.debug('cloud_init_wait ' + all_done) try: stdin, stdout, stderr = client.exec_command(all_done) stdout.channel.settimeout(5) out = stdout.read() log.debug('cloud_init_wait stdout ' + all_done + ' ' + out) except socket.timeout as e: client.close() log.debug('cloud_init_wait socket.timeout ' + all_done) continue except socket.error as e: client.close() log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done) continue log.debug('cloud_init_wait stderr ' + all_done + ' ' + stderr.read()) if stdout.channel.recv_exit_status() == 0: success = True client.close() if success: break return success
def test_and(self): ssh = connection.connect(HOST) r = run.run(client=ssh, args=["true", run.Raw("&&"), "echo", "yup"], stdout=StringIO()) assert r.stdout.getvalue() == "yup\n"
def cloud_init_wait(self, name_or_ip): """ Wait for cloud-init to complete on the name_or_ip OpenStack instance. """ log.debug('cloud_init_wait ' + name_or_ip) client_args = { 'user_at_host': '@'.join((self.username, name_or_ip)), 'timeout': 10, 'retry': False, } if self.key_filename: log.debug("using key " + self.key_filename) client_args['key_filename'] = self.key_filename with safe_while(sleep=2, tries=600, action="cloud_init_wait " + name_or_ip) as proceed: success = False # CentOS 6.6 logs in /var/log/clout-init-output.log # CentOS 7.0 logs in /var/log/clout-init.log all_done = ("tail /var/log/cloud-init*.log ; " + " test -f /tmp/init.out && tail /tmp/init.out ; " + " grep '" + self.up_string + "' " + "/var/log/cloud-init*.log") while proceed(): try: client = connection.connect(**client_args) except paramiko.PasswordRequiredException as e: raise Exception( "The private key requires a passphrase.\n" "Create a new key with:" " openstack keypair create myself > myself.pem\n" " chmod 600 myself.pem\n" "and call teuthology-openstack with the options\n" " --key-name myself --key-filename myself.pem\n") except paramiko.AuthenticationException as e: log.debug('cloud_init_wait AuthenticationException ' + str(e)) continue except socket.timeout as e: log.debug('cloud_init_wait connect socket.timeout ' + str(e)) continue except socket.error as e: log.debug('cloud_init_wait connect socket.error ' + str(e)) continue except Exception as e: transients = ('Incompatible ssh peer', 'Unknown server') for transient in transients: if transient in str(e): continue log.exception('cloud_init_wait ' + name_or_ip) raise log.debug('cloud_init_wait ' + all_done) try: stdin, stdout, stderr = client.exec_command(all_done) stdout.channel.settimeout(5) out = stdout.read() log.debug('cloud_init_wait stdout ' + all_done + ' ' + out) except socket.timeout as e: client.close() log.debug('cloud_init_wait socket.timeout ' + all_done) continue except socket.error as e: client.close() log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done) continue log.debug('cloud_init_wait stderr ' + all_done + ' ' + stderr.read()) if stdout.channel.recv_exit_status() == 0: success = True client.close() if success: break return success