def _is_reachable_via_ssh(self, ip_address, username, private_key,
timeout):
ssh_client = ssh.Client(ip_address,
username,
pkey=private_key,
timeout=timeout)
return ssh_client.test_connection_auth()
def test_get_ssh_connection(self):
c_mock, aa_mock, client_mock = self._set_ssh_connection_mocks()
s_mock = self.patch('time.sleep')
c_mock.return_value = client_mock
aa_mock.return_value = mock.sentinel.aa
# Test normal case for successful connection on first try
client = ssh.Client('localhost', 'root', timeout=2)
client._get_ssh_connection(sleep=1)
aa_mock.assert_called_once_with()
client_mock.set_missing_host_key_policy.assert_called_once_with(
mock.sentinel.aa)
expected_connect = [mock.call(
'localhost',
username='******',
pkey=None,
key_filename=None,
look_for_keys=False,
timeout=10.0,
password=None
)]
self.assertEqual(expected_connect, client_mock.connect.mock_calls)
s_mock.assert_not_called()
def _test_missing_package_sg3utils(self):
metric_name = "cinderDiagnostics.sg3utils"
error_injection_name = "available_package_sg3utils"
self.manager = self.os
#self.adm_manager= self.os_adm
self.image = CONF.compute.image_ref
self.flavor = CONF.compute.flavor_ref
#extra_specs = {"volume_backend_name":"3PAR-THEVERSE"}
#self.adm_manager.volume_types_client.create_volume_type(name='3PAR-THEVERSE', extra_specs=extra_specs)
# Step 1: create a volume
name = data_utils.rand_name("volume")
volume = self.manager.volumes_client.create_volume(
display_name=name, volume_type='3PAR-THEVERSE')
self.manager.volumes_client.wait_for_volume_status(
volume['id'], 'available')
# Test normal case for successful connection on first try
client = ssh.Client(CONF.cinder_diagnostics.cinder_ssh_ip,
CONF.cinder_diagnostics.cinder_ssh_user,
CONF.cinder_diagnostics.cinder_ssh_password,
timeout=30)
sshconnection = client._get_ssh_connection(sleep=1)
try:
# Setup sftp connection and transmit this script
sftp = sshconnection.open_sftp()
sftp.put("ErrorInjection.py", "ErrorInjection.py")
sftp.close()
client.exec_command(
"python ErrorInjection.py --missing_package_sg3utils")
sshconnection.close()
except IndexError:
pass
# Step 2: create vm instance
vm_name = data_utils.rand_name("instance")
server = self.manager.servers_client.create_server(
vm_name, self.image, self.flavor)
server_id = server['id']
waiters.wait_for_server_status(self.manager.servers_client, server_id,
'ACTIVE')
# Step 3: attach and detach volume to vm
self.manager.servers_client.attach_volume(server_id, volume['id'],
'/dev/vdc')
self.manager.volumes_client.wait_for_volume_status(
volume['id'], 'in-use')
self.manager.volumes_client.detach_volume(volume['id'])
self.manager.volumes_client.wait_for_volume_status(
volume['id'], 'available')
# Step 5: delete volume
self.manager.volumes_client.delete_volume(volume['id'])
self.manager.volumes_client.wait_for_resource_deletion(volume['id'])
# Step 4: delete vm
self.manager.servers_client.delete_server(server_id)
self.manager.servers_client.wait_for_server_termination(server_id)
self._test_cinder_diagnostics(metric_name, error_injection_name)
def do_ssh(command, host, ssh_user, ssh_key=None):
ssh_client = ssh.Client(host, ssh_user, key_filename=ssh_key)
try:
return ssh_client.exec_command(command)
except exceptions.SSHExecCommandFailed:
LOG.error('do_ssh raise exception. command:%s, host:%s.' %
(command, host))
return None
def test_pkey_calls_paramiko_RSAKey(self):
with contextlib.nested(
mock.patch('paramiko.RSAKey.from_private_key'),
mock.patch('cStringIO.StringIO')) as (rsa_mock, cs_mock):
cs_mock.return_value = mock.sentinel.csio
pkey = 'mykey'
ssh.Client('localhost', 'root', pkey=pkey)
rsa_mock.assert_called_once_with(mock.sentinel.csio)
cs_mock.assert_called_once_with('mykey')
rsa_mock.reset_mock()
cs_mock.rest_mock()
pkey = mock.sentinel.pkey
# Shouldn't call out to load a file from RSAKey, since
# a sentinel isn't a basestring...
ssh.Client('localhost', 'root', pkey=pkey)
rsa_mock.assert_not_called()
cs_mock.assert_not_called()
def do_ssh(command, host):
username = admin_manager.config.stress.target_ssh_user
key_filename = admin_manager.config.stress.target_private_key_path
if not (username and key_filename):
return None
ssh_client = ssh.Client(host, username, key_filename=key_filename)
try:
return ssh_client.exec_command(command)
except exceptions.SSHExecCommandFailed:
return None
def test_get_ssh_connection_two_attemps(self):
c_mock, aa_mock, client_mock = self._set_ssh_connection_mocks()
c_mock.return_value = client_mock
client_mock.connect.side_effect = [socket.error, mock.MagicMock()]
client = ssh.Client('localhost', 'root', timeout=1)
start_time = int(time.time())
client._get_ssh_connection(sleep=1)
end_time = int(time.time())
self.assertLess((end_time - start_time), 4)
self.assertGreater((end_time - start_time), 1)
def do_ssh(command, host):
username = admin_manager.config.stress.target_ssh_user
key_filename = admin_manager.config.stress.target_private_key_path
if not (username and key_filename):
LOG.error('username and key_filename should not be empty')
return None
ssh_client = ssh.Client(host, username, key_filename=key_filename)
try:
return ssh_client.exec_command(command)
except exceptions.SSHExecCommandFailed:
LOG.error('do_ssh raise exception. command:%s, host:%s.' %
(command, host))
return None
def test_get_ssh_connection_timeout(self):
c_mock, aa_mock, client_mock = self._set_ssh_connection_mocks()
c_mock.return_value = client_mock
client_mock.connect.side_effect = [
socket.error,
socket.error,
socket.error,
]
client = ssh.Client('localhost', 'root', timeout=2)
start_time = int(time.time())
with testtools.ExpectedException(exceptions.SSHTimeout):
client._get_ssh_connection()
end_time = int(time.time())
self.assertTrue((end_time - start_time) < 4)
self.assertTrue((end_time - start_time) >= 2)
def _start_servers(self):
"""
1. SSH to the instance
2. Start two servers listening on ports 80 and 88 respectively
"""
for server in self.servers_keypairs.keys():
ssh_login = config.compute.image_ssh_user
private_key = self.servers_keypairs[server].private_key
network_name = self.network.name
ip_address = server.networks[network_name][0]
ssh_client = ssh.Client(ip_address,
ssh_login,
pkey=private_key,
timeout=100)
start_server = "while true; do echo -e 'HTTP/1.0 200 OK\r\n\r\n" \
"%(server)s' | sudo nc -l -p %(port)s ; done &"
cmd = start_server % {'server': 'server1', 'port': self.port1}
ssh_client.exec_command(cmd)
cmd = start_server % {'server': 'server2', 'port': self.port2}
ssh_client.exec_command(cmd)
def __init__(self, server, username, password=None, pkey=None):
ssh_timeout = CONF.compute.ssh_timeout
network = CONF.compute.network_for_ssh
ip_version = CONF.compute.ip_version_for_ssh
ssh_channel_timeout = CONF.compute.ssh_channel_timeout
if isinstance(server, six.string_types):
ip_address = server
else:
addresses = server['addresses'][network]
for address in addresses:
if address['version'] == ip_version:
ip_address = address['addr']
break
else:
raise exceptions.ServerUnreachable()
self.ssh_client = ssh.Client(ip_address,
username,
password,
ssh_timeout,
pkey=pkey,
channel_timeout=ssh_channel_timeout)
def test_exec_command(self):
gsc_mock = self.patch('tempest.common.ssh.Client._get_ssh_connection')
ito_mock = self.patch('tempest.common.ssh.Client._is_timed_out')
select_mock = self.patch('select.poll')
client_mock = mock.MagicMock()
tran_mock = mock.MagicMock()
chan_mock = mock.MagicMock()
poll_mock = mock.MagicMock()
def reset_mocks():
gsc_mock.reset_mock()
ito_mock.reset_mock()
select_mock.reset_mock()
poll_mock.reset_mock()
client_mock.reset_mock()
tran_mock.reset_mock()
chan_mock.reset_mock()
select_mock.return_value = poll_mock
gsc_mock.return_value = client_mock
ito_mock.return_value = True
client_mock.get_transport.return_value = tran_mock
tran_mock.open_session.return_value = chan_mock
poll_mock.poll.side_effect = [
[0, 0, 0]
]
# Test for a timeout condition immediately raised
client = ssh.Client('localhost', 'root', timeout=2)
with testtools.ExpectedException(exceptions.TimeoutException):
client.exec_command("test")
chan_mock.fileno.assert_called_once_with()
chan_mock.exec_command.assert_called_once_with("test")
chan_mock.shutdown_write.assert_called_once_with()
SELECT_POLLIN = 1
poll_mock.register.assert_called_once_with(chan_mock, SELECT_POLLIN)
poll_mock.poll.assert_called_once_with(10)
# Test for proper reading of STDOUT and STDERROR and closing
# of all file descriptors.
reset_mocks()
select_mock.return_value = poll_mock
gsc_mock.return_value = client_mock
ito_mock.return_value = False
client_mock.get_transport.return_value = tran_mock
tran_mock.open_session.return_value = chan_mock
poll_mock.poll.side_effect = [
[1, 0, 0]
]
closed_prop = mock.PropertyMock(return_value=True)
type(chan_mock).closed = closed_prop
chan_mock.recv_exit_status.return_value = 0
chan_mock.recv.return_value = ''
chan_mock.recv_stderr.return_value = ''
client = ssh.Client('localhost', 'root', timeout=2)
client.exec_command("test")
chan_mock.fileno.assert_called_once_with()
chan_mock.exec_command.assert_called_once_with("test")
chan_mock.shutdown_write.assert_called_once_with()
SELECT_POLLIN = 1
poll_mock.register.assert_called_once_with(chan_mock, SELECT_POLLIN)
poll_mock.poll.assert_called_once_with(10)
chan_mock.recv_ready.assert_called_once_with()
chan_mock.recv.assert_called_once_with(1024)
chan_mock.recv_stderr_ready.assert_called_once_with()
chan_mock.recv_stderr.assert_called_once_with(1024)
chan_mock.recv_exit_status.assert_called_once_with()
closed_prop.assert_called_once_with()
def _test_cinder_diagnostics(self, metric_name, error_injection_name):
m_starttime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
m_starttime = m_starttime.replace(' ', 'T') + 'Z'
# Test case to check if new notification is created successfully.
notification_name = data_utils.rand_name('notification-')
notification_type = 'EMAIL'
# Replace below email with valid email address as required.
u_address = 'root@localhost'
response = self.monitoring_client.create_notification(
name=notification_name, type=notification_type, address=u_address)
self.assertEqual(notification_name, response['name'])
notification_id = response['id']
# Delete notification
#alarm_def_name = metric_name
alarm_def_name = data_utils.rand_name('test_monasca_alarm_definition')
expression = '%s > 0' % (metric_name)
severity = 'HIGH'
body = self.monitoring_client.create_alarm_definition(
name=alarm_def_name, expression=expression, severity=severity)
self.assertEqual(alarm_def_name, body['name'])
alarm_def_id = body['id']
self.assertEqual(expression, body['expression'])
m_statistics = 'count'
m_endtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
m_endtime = m_endtime.replace(' ', 'T') + 'Z'
body = self.monitoring_client.metric_statistics(
name=metric_name,
dimensions='service:cinder',
statistics=m_statistics,
end_time=m_endtime,
merge_metrics='true')
self.assertEqual('200', body.response['status'])
response = json.loads(body.data)
# Test normal case for successful connection on first try
client = ssh.Client(CONF.cinder_diagnostics.cinder_ssh_ip,
CONF.cinder_diagnostics.cinder_ssh_user,
CONF.cinder_diagnostics.cinder_ssh_password,
timeout=100)
sshconnection = client._get_ssh_connection(sleep=1)
try:
# Setup sftp connection and transmit this script
sftp = sshconnection.open_sftp()
sftp.put("ErrorInjection.py", "ErrorInjection.py")
sftp.close()
except IndexError:
pass
client.exec_command("python ErrorInjection.py --" +
error_injection_name)
isMetricAvailable = True
for i in range(0, 30):
metricFields = {'name': metric_name}
#metricFields['timestamp'] = int((time.time()*1000))
body = self.monitoring_client.list_metric(metricFields)
self.assertEqual('200', body.response['status'])
response = json.loads(body.data)
if len(response['elements']) < 1:
isMetricAvailable = False
isMeasurementsFound = False
isAlarmFound = False
isStateChangeToAlarm = False
for i in range(0, 30):
m_statistics = 'count'
m_endtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
m_endtime = m_endtime.replace(' ', 'T') + 'Z'
body = self.monitoring_client.metric_statistics(
name=metric_name,
dimensions='service:cinder',
statistics=m_statistics,
start_time=m_starttime,
end_time=m_endtime,
merge_metrics='true')
self.assertEqual('200', body.response['status'])
response = json.loads(body.data)
if len(response['elements']) > 0:
if len(response['elements'][0]['statistics']) == 1:
isMeasurementsFound = True
break
for i in range(0, 60):
body = self.monitoring_client.list_alarms(
alarm_definition_id=alarm_def_id)
if len(body['elements']) > 0:
isAlarmFound = True
if body['elements'][0]['state'] == 'ALARM':
isStateChangeToAlarm = True
break
time.sleep(3)
self.monitoring_client.delete_notification(notification_id)
self.monitoring_client.delete_alarm_definition(alarm_def_id)
if not isMetricAvailable:
self.fail("No metric found " + metric_name)
if not isMeasurementsFound and not isStateChangeToAlarm:
self.fail("No measurements found for " + metric_name)
if not isAlarmFound:
self.fail("No alarm found for " + metric_name)
if not isStateChangeToAlarm:
self.fail("No state change to ALARM for metric " + metric_name)
def test_get_ssh_connection(self):
c_mock = self.patch('paramiko.SSHClient')
aa_mock = self.patch('paramiko.AutoAddPolicy')
s_mock = self.patch('time.sleep')
t_mock = self.patch('time.time')
aa_mock.return_value = mock.sentinel.aa
def reset_mocks():
aa_mock.reset_mock()
c_mock.reset_mock()
s_mock.reset_mock()
t_mock.reset_mock()
# Test normal case for successful connection on first try
client_mock = mock.MagicMock()
c_mock.return_value = client_mock
client_mock.connect.return_value = True
client = ssh.Client('localhost', 'root', timeout=2)
client._get_ssh_connection(sleep=1)
aa_mock.assert_called_once_with()
client_mock.set_missing_host_key_policy.assert_called_once_with(
mock.sentinel.aa)
expected_connect = [
mock.call('localhost',
username='******',
pkey=None,
key_filename=None,
look_for_keys=False,
timeout=10.0,
password=None)
]
self.assertEqual(expected_connect, client_mock.connect.mock_calls)
s_mock.assert_not_called()
t_mock.assert_called_once_with()
reset_mocks()
# Test case when connection fails on first two tries and
# succeeds on third try (this validates retry logic)
client_mock.connect.side_effect = [socket.error, socket.error, True]
t_mock.side_effect = [
1000, # Start time
1001, # Sleep loop 1
1002 # Sleep loop 2
]
client._get_ssh_connection(sleep=1)
expected_sleeps = [mock.call(1), mock.call(1.01)]
self.assertEqual(expected_sleeps, s_mock.mock_calls)
reset_mocks()
# Test case when connection fails on first three tries and
# exceeds the timeout, so expect to raise a Timeout exception
client_mock.connect.side_effect = [
socket.error, socket.error, socket.error
]
t_mock.side_effect = [
1000, # Start time
1001, # Sleep loop 1
1002, # Sleep loop 2
1003, # Sleep loop 3
1004 # LOG.error() calls time.time()
]
with testtools.ExpectedException(exceptions.SSHTimeout):
client._get_ssh_connection()
