def ssh(self, command):
"""
Run a command in the OpenStack instance of the teuthology cluster.
Return the stdout / stderr of the command.
"""
instance_id = self.get_instance_id(self.args.name)
ip = self.get_floating_ip_or_ip(instance_id)
client_args = {
'user_at_host': '@'.join((self.username, ip)),
'retry': False,
}
if self.key_filename:
log.debug("ssh overriding key with " + self.key_filename)
client_args['key_filename'] = self.key_filename
client = connection.connect(**client_args)
stdin, stdout, stderr = client.exec_command(command)
stdout.channel.settimeout(300)
out = ''
try:
out = stdout.read()
log.debug('teardown stdout ' + command + ' ' + out)
except Exception:
log.exception('teardown ' + command + ' failed')
err = stderr.read()
log.debug('teardown stderr ' + command + ' ' + err)
return out + ' ' + err
def ssh(self, command):
"""
Run a command in the OpenStack instance of the teuthology cluster.
Return the stdout / stderr of the command.
"""
ip = self.instance.get_floating_ip_or_ip()
client_args = {
'user_at_host': '@'.join((self.username, ip)),
'retry': False,
'timeout': 240,
}
if self.key_filename:
log.debug("ssh overriding key with " + self.key_filename)
client_args['key_filename'] = self.key_filename
client = connection.connect(**client_args)
# get the I/O channel to iterate line by line
transport = client.get_transport()
channel = transport.open_session()
channel.get_pty()
channel.settimeout(900)
output = channel.makefile('r', 1)
log.debug(":ssh@" + ip + ":" + command)
channel.exec_command(command)
for line in iter(output.readline, b''):
log.info(line.strip())
return channel.recv_exit_status()
def test_connect_override_hostkeys(self):
self.clear_config()
m_ssh_instance = self.m_ssh.return_value = Mock()
m_transport = Mock()
m_ssh_instance.get_transport.return_value = m_transport
m_host_keys = Mock()
m_ssh_instance.get_host_keys.return_value = m_host_keys
m_create_key = Mock()
m_create_key.return_value = "frobnitz"
got = connection.connect(
'*****@*****.**',
host_key='ssh-rsa testkey',
_SSHClient=self.m_ssh,
_create_key=m_create_key,
)
self.m_ssh.assert_called_once()
m_ssh_instance.get_host_keys.assert_called_once()
m_host_keys.add.assert_called_once_with(
hostname='orchestra.test.newdream.net.invalid',
keytype='ssh-rsa',
key='frobnitz',
)
m_create_key.assert_called_once_with('ssh-rsa', 'testkey')
m_ssh_instance.connect.assert_called_once_with(
hostname='orchestra.test.newdream.net.invalid',
username='******',
timeout=60,
)
m_transport.set_keepalive.assert_called_once_with(False)
assert got is m_ssh_instance
def ssh(self, command):
"""
Run a command in the OpenStack instance of the teuthology cluster.
Return the stdout / stderr of the command.
"""
instance_id = self.get_instance_id(self.args.name)
ip = self.get_floating_ip_or_ip(instance_id)
client_args = {
'user_at_host': '@'.join((self.username, ip)),
'retry': False,
'timeout': 240,
}
if self.key_filename:
log.debug("ssh overriding key with " + self.key_filename)
client_args['key_filename'] = self.key_filename
client = connection.connect(**client_args)
stdin, stdout, stderr = client.exec_command(command)
stdout.channel.settimeout(300)
out = ''
try:
out = stdout.read()
log.debug('ssh stdout ' + command + ' ' + out)
except Exception:
log.exception('ssh ' + command + ' failed')
err = stderr.read()
log.debug('ssh stderr ' + command + ' ' + err)
return out + ' ' + err
def ssh(self, command):
"""
Run a command in the OpenStack instance of the teuthology cluster.
Return the stdout / stderr of the command.
"""
ip = self.instance.get_floating_ip_or_ip()
client_args = {
'user_at_host': '@'.join((self.username, ip)),
'retry': False,
'timeout': 240,
}
if self.key_filename:
log.debug("ssh overriding key with " + self.key_filename)
client_args['key_filename'] = self.key_filename
client = connection.connect(**client_args)
# get the I/O channel to iterate line by line
transport = client.get_transport()
channel = transport.open_session()
channel.get_pty()
channel.settimeout(900)
output = channel.makefile('r', 1)
log.debug(":ssh@" + ip + ":" + command)
channel.exec_command(command)
for line in iter(output.readline, b''):
log.info(line.strip())
return channel.recv_exit_status()
def test_connect_override_hostkeys(self):
self.clear_config()
fudge.clear_expectations()
sshclient = fudge.Fake('SSHClient')
ssh = sshclient.expects_call().with_args().returns_fake()
ssh.remember_order()
host_keys = fudge.Fake('HostKeys')
host_keys.expects('add').with_args(
hostname='orchestra.test.newdream.net.invalid',
keytype='ssh-rsa',
key='frobnitz',
)
ssh.expects('get_host_keys').with_args().returns(host_keys)
ssh.expects('connect').with_args(
hostname='orchestra.test.newdream.net.invalid',
username='******',
timeout=60,
)
transport = ssh.expects('get_transport').with_args().returns_fake()
transport.remember_order()
transport.expects('set_keepalive').with_args(False)
create_key = fudge.Fake('create_key')
create_key.expects_call().with_args('ssh-rsa',
'testkey').returns('frobnitz')
got = connection.connect(
'*****@*****.**',
host_key='ssh-rsa testkey',
_SSHClient=sshclient,
_create_key=create_key,
)
assert got is ssh
def test_and(self):
ssh = connection.connect(HOST)
r = run.run(
client=ssh,
args=['true', run.Raw('&&'), 'echo', 'yup'],
stdout=StringIO(),
)
assert r.stdout.getvalue() == 'yup\n'
def test_and(self):
ssh = connection.connect(HOST)
r = run.run(
client=ssh,
args=['true', run.Raw('&&'), 'echo', 'yup'],
stdout=StringIO(),
)
assert r.stdout.getvalue() == 'yup\n'
def test_crash(self):
ssh = connection.connect(HOST)
e = assert_raises(
CommandCrashedError,
run.run,
client=ssh,
args=['sh', '-c', 'kill -ABRT $$'],
)
assert e.command == "sh -c 'kill -ABRT $$'"
assert str(e) == "Command crashed: \"sh -c 'kill -ABRT $$'\""
def test_crash(self):
ssh = connection.connect(HOST)
e = assert_raises(
CommandCrashedError,
run.run,
client=ssh,
args=['sh', '-c', 'kill -ABRT $$'],
)
assert e.command == "sh -c 'kill -ABRT $$'"
assert str(e) == "Command crashed: \"sh -c 'kill -ABRT $$'\""
def test_lost(self):
ssh = connection.connect(HOST)
e = assert_raises(
ConnectionLostError,
run.run,
client=ssh,
args=['sh', '-c', 'kill -ABRT $PPID'],
)
assert e.command == "sh -c 'kill -ABRT $PPID'"
assert str(e) == \
"SSH connection was lost: \"sh -c 'kill -ABRT $PPID'\""
def test_pipe(self):
ssh = connection.connect(HOST)
r = run.run(client=ssh, args=["cat"], stdin=run.PIPE, stdout=StringIO(), wait=False)
assert r.stdout.getvalue() == ""
r.stdin.write("foo\n")
r.stdin.write("bar\n")
r.stdin.close()
r.wait()
got = r.exitstatus
assert got == 0
assert r.stdout.getvalue() == "foo\nbar\n"
def test_lost(self):
ssh = connection.connect(HOST)
e = assert_raises(
ConnectionLostError,
run.run,
client=ssh,
args=['sh', '-c', 'kill -ABRT $PPID'],
name=HOST,
)
assert e.command == "sh -c 'kill -ABRT $PPID'"
assert str(e) == \
"SSH connection to {host} was lost: ".format(host=HOST) + \
"\"sh -c 'kill -ABRT $PPID'\""
def test_lost(self):
ssh = connection.connect(HOST)
e = assert_raises(
ConnectionLostError,
run.run,
client=ssh,
args=['sh', '-c', 'kill -ABRT $PPID'],
name=HOST,
)
assert e.command == "sh -c 'kill -ABRT $PPID'"
assert str(e) == \
"SSH connection to {host} was lost: ".format(host=HOST) + \
"\"sh -c 'kill -ABRT $PPID'\""
def connect(self, timeout=None, create_key=None, context='connect'):
args = dict(user_at_host=self.name, host_key=self._host_key,
keep_alive=self.keep_alive, _create_key=create_key)
if context == 'reconnect':
# The reason for the 'context' workaround is not very
# clear from the technical side.
# I'll get "[Errno 98] Address already in use" altough
# there are no open tcp(ssh) connections.
# When connecting without keepalive, host_key and _create_key
# set, it will proceed.
args = dict(user_at_host=self.name, _create_key=False, host_key=None)
if timeout:
args['timeout'] = timeout
self.ssh = connection.connect(**args)
return self.ssh
def test_pipe(self):
ssh = connection.connect(HOST)
r = run.run(
client=ssh,
args=['cat'],
stdin=run.PIPE,
stdout=StringIO(),
wait=False,
)
assert r.stdout.getvalue() == ''
r.stdin.write('foo\n')
r.stdin.write('bar\n')
r.stdin.close()
r.wait()
got = r.exitstatus
assert got == 0
assert r.stdout.getvalue() == 'foo\nbar\n'
def test_connect_no_verify_host_keys(self):
self.clear_config()
config.config.verify_host_keys = False
fudge.clear_expectations()
ssh = fudge.Fake('SSHClient')
ssh.expects_call().with_args().returns(ssh)
ssh.expects('set_missing_host_key_policy')
ssh.expects('connect').with_args(
hostname='orchestra.test.newdream.net.invalid',
username='******',
timeout=60,
)
transport = ssh.expects('get_transport').with_args().returns_fake()
transport.remember_order()
transport.expects('set_keepalive').with_args(False)
got = connection.connect(
'*****@*****.**',
_SSHClient=ssh,
)
assert got is ssh
def test_connect_no_verify_host_keys(self):
self.clear_config()
config.config.verify_host_keys = False
m_ssh_instance = self.m_ssh.return_value = Mock()
m_transport = Mock()
m_ssh_instance.get_transport.return_value = m_transport
got = connection.connect(
'*****@*****.**',
_SSHClient=self.m_ssh,
)
self.m_ssh.assert_called_once()
m_ssh_instance.set_missing_host_key_policy.assert_called_once()
assert not m_ssh_instance.load_system_host_keys.called
m_ssh_instance.connect.assert_called_once_with(
hostname='orchestra.test.newdream.net.invalid',
username='******',
timeout=60,
)
m_transport.set_keepalive.assert_called_once_with(False)
assert got is m_ssh_instance
def cloud_init_wait(self, instance):
"""
Wait for cloud-init to complete on the name_or_ip OpenStack instance.
"""
ip = instance.get_floating_ip_or_ip()
log.debug('cloud_init_wait ' + ip)
client_args = {
'user_at_host': '@'.join((self.username, ip)),
'timeout': 240,
'retry': False,
}
if self.key_filename:
log.debug("using key " + self.key_filename)
client_args['key_filename'] = self.key_filename
with safe_while(sleep=30, tries=30,
action="cloud_init_wait " + ip) as proceed:
success = False
# CentOS 6.6 logs in /var/log/clout-init-output.log
# CentOS 7.0 logs in /var/log/clout-init.log
tail = ("tail --follow=name --retry"
" /var/log/cloud-init*.log /tmp/init.out")
while proceed():
try:
client = connection.connect(**client_args)
except paramiko.PasswordRequiredException as e:
raise Exception(
"The private key requires a passphrase.\n"
"Create a new key with:"
" openstack keypair create myself > myself.pem\n"
" chmod 600 myself.pem\n"
"and call teuthology-openstack with the options\n"
" --key-name myself --key-filename myself.pem\n")
except paramiko.AuthenticationException as e:
log.debug('cloud_init_wait AuthenticationException ' + str(e))
continue
except socket.timeout as e:
log.debug('cloud_init_wait connect socket.timeout ' + str(e))
continue
except socket.error as e:
log.debug('cloud_init_wait connect socket.error ' + str(e))
continue
except Exception as e:
transients = ('Incompatible ssh peer', 'Unknown server')
for transient in transients:
if transient in str(e):
continue
log.exception('cloud_init_wait ' + ip)
raise
log.debug('cloud_init_wait ' + tail)
try:
# get the I/O channel to iterate line by line
transport = client.get_transport()
channel = transport.open_session()
channel.get_pty()
channel.settimeout(240)
output = channel.makefile('r', 1)
channel.exec_command(tail)
for line in iter(output.readline, b''):
log.info(line.strip())
if self.up_string in line:
success = True
break
except socket.timeout as e:
client.close()
log.debug('cloud_init_wait socket.timeout ' + tail)
continue
except socket.error as e:
client.close()
log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + tail)
continue
client.close()
if success:
break
return success
def cloud_init_wait(self, instance):
"""
Wait for cloud-init to complete on the name_or_ip OpenStack instance.
"""
ip = instance.get_floating_ip_or_ip()
log.debug('cloud_init_wait ' + ip)
client_args = {
'user_at_host': '@'.join((self.username, ip)),
'timeout': 240,
'retry': False,
}
if self.key_filename:
log.debug("using key " + self.key_filename)
client_args['key_filename'] = self.key_filename
with safe_while(sleep=30, tries=30,
action="cloud_init_wait " + ip) as proceed:
success = False
# CentOS 6.6 logs in /var/log/clout-init-output.log
# CentOS 7.0 logs in /var/log/clout-init.log
tail = ("tail --follow=name --retry"
" /var/log/cloud-init*.log /tmp/init.out")
while proceed():
try:
client = connection.connect(**client_args)
except paramiko.PasswordRequiredException as e:
raise Exception(
"The private key requires a passphrase.\n"
"Create a new key with:"
" openstack keypair create myself > myself.pem\n"
" chmod 600 myself.pem\n"
"and call teuthology-openstack with the options\n"
" --key-name myself --key-filename myself.pem\n")
except paramiko.AuthenticationException as e:
log.debug('cloud_init_wait AuthenticationException ' +
str(e))
continue
except socket.timeout as e:
log.debug('cloud_init_wait connect socket.timeout ' +
str(e))
continue
except socket.error as e:
log.debug('cloud_init_wait connect socket.error ' + str(e))
continue
except Exception as e:
transients = ('Incompatible ssh peer', 'Unknown server')
for transient in transients:
if transient in str(e):
continue
log.exception('cloud_init_wait ' + ip)
raise
log.debug('cloud_init_wait ' + tail)
try:
# get the I/O channel to iterate line by line
transport = client.get_transport()
channel = transport.open_session()
channel.get_pty()
channel.settimeout(240)
output = channel.makefile('r', 1)
channel.exec_command(tail)
for line in iter(output.readline, b''):
log.info(line.strip())
if self.up_string in line:
success = True
break
except socket.timeout as e:
client.close()
log.debug('cloud_init_wait socket.timeout ' + tail)
continue
except socket.error as e:
client.close()
log.debug('cloud_init_wait socket.error ' + str(e) + ' ' +
tail)
continue
client.close()
if success:
break
return success
def cloud_init_wait(self, name_or_ip):
"""
Wait for cloud-init to complete on the name_or_ip OpenStack instance.
"""
log.debug('cloud_init_wait ' + name_or_ip)
client_args = {
'user_at_host': '@'.join((self.username, name_or_ip)),
'timeout': 240,
'retry': False,
}
if self.key_filename:
log.debug("using key " + self.key_filename)
client_args['key_filename'] = self.key_filename
with safe_while(sleep=30, tries=100,
action="cloud_init_wait " + name_or_ip) as proceed:
success = False
# CentOS 6.6 logs in /var/log/clout-init-output.log
# CentOS 7.0 logs in /var/log/clout-init.log
all_done = ("tail /var/log/cloud-init*.log ; " +
" test -f /tmp/init.out && tail /tmp/init.out ; " +
" grep '" + self.up_string + "' " +
"/var/log/cloud-init*.log")
while proceed():
try:
client = connection.connect(**client_args)
except paramiko.PasswordRequiredException as e:
raise Exception(
"The private key requires a passphrase.\n"
"Create a new key with:"
" openstack keypair create myself > myself.pem\n"
" chmod 600 myself.pem\n"
"and call teuthology-openstack with the options\n"
" --key-name myself --key-filename myself.pem\n")
except paramiko.AuthenticationException as e:
log.debug('cloud_init_wait AuthenticationException ' + str(e))
continue
except socket.timeout as e:
log.debug('cloud_init_wait connect socket.timeout ' + str(e))
continue
except socket.error as e:
log.debug('cloud_init_wait connect socket.error ' + str(e))
continue
except Exception as e:
transients = ('Incompatible ssh peer', 'Unknown server')
for transient in transients:
if transient in str(e):
continue
log.exception('cloud_init_wait ' + name_or_ip)
raise
log.debug('cloud_init_wait ' + all_done)
try:
stdin, stdout, stderr = client.exec_command(all_done)
stdout.channel.settimeout(5)
out = stdout.read()
log.debug('cloud_init_wait stdout ' + all_done + ' ' + out)
except socket.timeout as e:
client.close()
log.debug('cloud_init_wait socket.timeout ' + all_done)
continue
except socket.error as e:
client.close()
log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done)
continue
log.debug('cloud_init_wait stderr ' + all_done +
' ' + stderr.read())
if stdout.channel.recv_exit_status() == 0:
success = True
client.close()
if success:
break
return success
def test_and(self):
ssh = connection.connect(HOST)
r = run.run(client=ssh, args=["true", run.Raw("&&"), "echo", "yup"], stdout=StringIO())
assert r.stdout.getvalue() == "yup\n"
def cloud_init_wait(self, name_or_ip):
"""
Wait for cloud-init to complete on the name_or_ip OpenStack instance.
"""
log.debug('cloud_init_wait ' + name_or_ip)
client_args = {
'user_at_host': '@'.join((self.username, name_or_ip)),
'timeout': 10,
'retry': False,
}
if self.key_filename:
log.debug("using key " + self.key_filename)
client_args['key_filename'] = self.key_filename
with safe_while(sleep=2, tries=600,
action="cloud_init_wait " + name_or_ip) as proceed:
success = False
# CentOS 6.6 logs in /var/log/clout-init-output.log
# CentOS 7.0 logs in /var/log/clout-init.log
all_done = ("tail /var/log/cloud-init*.log ; " +
" test -f /tmp/init.out && tail /tmp/init.out ; " +
" grep '" + self.up_string + "' " +
"/var/log/cloud-init*.log")
while proceed():
try:
client = connection.connect(**client_args)
except paramiko.PasswordRequiredException as e:
raise Exception(
"The private key requires a passphrase.\n"
"Create a new key with:"
" openstack keypair create myself > myself.pem\n"
" chmod 600 myself.pem\n"
"and call teuthology-openstack with the options\n"
" --key-name myself --key-filename myself.pem\n")
except paramiko.AuthenticationException as e:
log.debug('cloud_init_wait AuthenticationException ' + str(e))
continue
except socket.timeout as e:
log.debug('cloud_init_wait connect socket.timeout ' + str(e))
continue
except socket.error as e:
log.debug('cloud_init_wait connect socket.error ' + str(e))
continue
except Exception as e:
transients = ('Incompatible ssh peer', 'Unknown server')
for transient in transients:
if transient in str(e):
continue
log.exception('cloud_init_wait ' + name_or_ip)
raise
log.debug('cloud_init_wait ' + all_done)
try:
stdin, stdout, stderr = client.exec_command(all_done)
stdout.channel.settimeout(5)
out = stdout.read()
log.debug('cloud_init_wait stdout ' + all_done + ' ' + out)
except socket.timeout as e:
client.close()
log.debug('cloud_init_wait socket.timeout ' + all_done)
continue
except socket.error as e:
client.close()
log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done)
continue
log.debug('cloud_init_wait stderr ' + all_done +
' ' + stderr.read())
if stdout.channel.recv_exit_status() == 0:
success = True
client.close()
if success:
break
return success
