def get_ssh_connection_to_controller(self, controller):
remote = ssh.Client(host=controller,
username=self._usr,
password=self._pwd,
key_filename=self._key,
timeout=self._ssh_timeout)
return remote
def test_001_check_state_of_backends(self):
"""Check state of haproxy backends on controllers
Target Service: HA haproxy
Scenario:
1. Ssh on each controller and get state of HAProxy backends
2. Check backend state for availability
Duration: 10 s.
Available since release: 2015.1.0-8.0
"""
LOG.info("Controllers nodes are %s" % self.controllers)
for controller in self.controllers:
remote = ssh.Client(controller,
self.controller_user,
key_filename=self.controller_key,
timeout=100)
ignore_services = []
if 'neutron' not in self.config.network.network_provider:
ignore_services.append('nova-metadata-api')
haproxy_status = self.verify(10,
self._check_haproxy_backend,
1,
"Can't get state of backends.",
"Getting state of backends",
remote,
ignore_services=ignore_services)
dead_backends = filter(lambda x: 'DOWN' in x,
haproxy_status.splitlines())
backends_message = "Dead backends {0}"\
.format(dead_backends)
LOG.debug(backends_message)
error_message = "Step 2 failed: " + backends_message
self.verify_response_true(len(dead_backends) == 0, error_message)
def test_002_internet_connectivity_from_compute(self):
"""Check internet connectivity from a compute
Target component: OpenStack
Scenario:
1. Execute ping 8.8.8.8 command from a compute node.
Duration: 100 s.
Deployment tags: qemu | kvm, public_on_all_nodes | nova_network
"""
if not self.computes:
self.skipTest('There are no compute nodes')
cmd = "ping -q -c1 -w10 8.8.8.8"
ssh_client = ssh.Client(self.computes[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
self.verify(100, self.retry_command, 1,
"'ping' command failed. Looks like there is no "
"Internet connection on the compute node.",
"'ping' command",
2, 30, ssh_client.exec_command, cmd)
def tearDownClass(cls):
if cls.master_ip:
try:
cmd = "mysql -h localhost -e 'DROP DATABASE %s'" % cls.database
ssh.Client(cls.master_ip,
cls.node_user,
key_filename=cls.node_key).exec_command(cmd)
except Exception:
LOG.exception("Failed to connect to mysql cmd:{0}".format(cmd))
def _run_ssh_cmd(self, host, cmd):
"""Open SSH session with host and and execute command."""
try:
sshclient = ssh.Client(host, self.controller_user,
self.controllers_pwd,
key_filename=self.controller_key,
timeout=self.timeout)
return sshclient.exec_longrun_command(cmd)
except Exception:
LOG.debug(traceback.format_exc())
self.fail("%s command failed." % cmd)
def _run_ssh_cmd(self, host, cmd):
"""Open SSH session with host and execute command."""
try:
sshclient = ssh.Client(host,
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
return sshclient.exec_longrun_command(cmd)
except Exception:
LOG.exception('Failure on ssh run cmd')
self.fail("%s command failed." % cmd)
def test_os_databases(self):
"""Check if amount of tables in databases is the same on each node
Target Service: HA mysql
Scenario:
1. Detect there are online database nodes.
2. Request list of tables for os databases on each node.
3. Check if amount of tables in databases is the same on each node
Duration: 10 s.
"""
LOG.info("'Test OS Databases' started")
dbs = ['nova', 'glance', 'keystone']
cmd = "mysql -h localhost -e 'SHOW TABLES FROM %(database)s'"
databases = self.verify(20,
self.get_database_nodes,
1, "Can not get database hostnames. Check that"
" at least one controller is operable",
"get database nodes",
self.controller_ip,
self.node_user,
key=self.node_key)
self.verify_response_body_not_equal(0, len(databases), self.no_db_msg,
1)
if len(databases) == 1:
self.skipTest(self.one_db_msg)
for database in dbs:
LOG.info('Current database name is %s' % database)
temp_set = set()
for node in databases:
LOG.info('Current database node is %s' % node)
cmd1 = cmd % {'database': database}
LOG.info('Try to execute command %s' % cmd1)
tables = ssh.Client(node,
self.node_user,
key_filename=self.node_key,
timeout=self.config.compute.ssh_timeout)
output = self.verify(40, tables.exec_command, 2,
'Can list tables',
'get amount of tables for each database',
cmd1)
tables = set(output.splitlines())
if len(temp_set) == 0:
temp_set = tables
self.verify_response_true(
len(tables.symmetric_difference(temp_set)) == 0,
"Step 3 failed: Tables in %s database are "
"different" % database)
del temp_set
def run_ssh_cmd_with_exit_code(self, host, cmd):
"""Open SSH session with host and execute command.
Fail if exit code != 0
"""
try:
sshclient = ssh.Client(host,
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
return sshclient.exec_command(cmd)
except Exception:
LOG.exception("Failed while opening ssh session with host")
self.fail("{0} command failed.".format(cmd))
def test_003_dns_resolution(self):
"""Check DNS resolution on compute node
Target component: OpenStack
Scenario:
1. Execute host 8.8.8.8 command from a compute node.
2. Check 8.8.8.8 host was successfully resolved
3. Check host google.com command from the compute node.
4. Check google.com host was successfully resolved.
Duration: 120 s.
Deployment tags: qemu | kvm, public_on_all_nodes | nova_network
"""
if not self.computes:
self.skipTest('There are no computes nodes')
dns = self.fuel_dns.spit(',') if self.fuel_dns else ['8.8.8.8']
ssh_client = ssh.Client(self.computes[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
expected_output = "{0}.in-addr.arpa domain name pointer".format(dns[0])
cmd = "host {0}".format(dns[0])
output = self.verify(100, self.retry_command, 1,
"'host' command failed. Looks like there is no "
"Internet connection on the computes node.",
"'ping' command", 10, 5,
ssh_client.exec_command, cmd)
LOG.debug(output)
self.verify_response_true(expected_output in output,
'Step 2 failed: '
'DNS name for {0} host '
'cannot be resolved.'.format(dns[0]))
domain_name = output.split()[-1]
cmd = "host {0}".format(domain_name)
output = self.verify(100, self.retry_command, 3,
"'host' command failed. "
"DNS name cannot be resolved.",
"'host' command", 10, 5,
ssh_client.exec_command, cmd)
LOG.debug(output)
self.verify_response_true('has address {0}'.format(dns[0]) in output,
'Step 4 failed: '
'DNS name cannot be resolved.')
def check_services():
succeed_count = 0
for node in nodes:
remote = ssh.Client(node,
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
try:
output = remote.exec_command(cmd)
LOG.debug(output)
if expected in output:
succeed_count += 1
except Exception:
pass
if succeed_count == succeed_nodes:
return True
else:
return False
def test_001_check_default_master_node_credential_usage(self):
"""Check usage of default credentials on master node
Target component: Configuration
Scenario:
1. Check user can not ssh on master node with default credentials.
Duration: 20 s.
Available since release: 2014.2-6.1
"""
ip = self.config.nailgun_host
ssh_client = ssh.Client(ip,
self.config.master.master_node_ssh_user,
self.config.master.master_node_ssh_password,
timeout=self.config.master.ssh_timeout)
cmd = "date"
output = []
try:
output = ssh_client.exec_command(cmd)
LOG.debug(output)
except exceptions.SSHExecCommandFailed:
self.verify_response_true(len(output) == 0,
'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
except exceptions.TimeoutException:
self.verify_response_true(len(output) == 0,
'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
except exc.SSHException:
self.verify_response_true(len(output) == 0,
'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
self.verify_response_true(len(output) == 0,
'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
def get_database_nodes(cls, controller_ip, username, key):
if version.StrictVersion(cls.release_version)\
< version.StrictVersion('7.0'):
return cls.config.compute.online_controllers
# retrieve data from controller
ssh_client = ssh.Client(controller_ip,
username,
key_filename=key,
timeout=100)
hiera_cmd = ('ruby -e \'require "hiera"; '
'db_h = Hiera.new().lookup("database_nodes", {}, {}); '
'db = db_h.keys.map{|k| db_h[k]["name"]}; '
'if db != [] then puts db else puts "None" end\'')
database_nodes = ssh_client.exec_command(hiera_cmd)
# get online nodes
database_nodes = database_nodes.splitlines()
databases = []
for node in cls.config.compute.nodes:
hostname = node['hostname']
if hostname in database_nodes and node['online']:
databases.append(hostname)
return databases
def test_mysql_replication(self):
"""Check data replication over mysql
Target Service: HA mysql
Scenario:
1. Check that mysql is running on all controller or database nodes.
2. Create database on one node.
3. Create table in created database
4. Insert data to the created table
5. Get replicated data from each database node.
6. Verify that replicated data in the same from each database
7. Drop created database
Duration: 10 s.
"""
LOG.info("'Test MySQL replication' started")
databases = self.verify(20,
self.get_database_nodes,
1, "Can not get database hostnames. Check that"
" at least one controller is operable",
"get database nodes",
self.controller_ip,
self.node_user,
key=self.node_key)
self.verify_response_body_not_equal(0, len(databases), self.no_db_msg,
1)
if len(databases) == 1:
self.skipTest(self.one_db_msg)
LOG.info("Database nodes are " + ", ".join(databases))
self.master_ip = databases[0]
# check that mysql is running on all hosts
cmd = 'mysql -h localhost -e "" '
for db_node in databases:
ssh_client = ssh.Client(db_node,
self.node_user,
key_filename=self.node_key,
timeout=100)
self.verify(
20, ssh_client.exec_command, 1, 'Can not connect to mysql. '
'Please check that mysql is running and there '
'is connectivity by management network', 'detect mysql node',
cmd)
database_name = self.database
table_name = 'ost' + str(data_utils.rand_int_id(100, 999))
record_data = str(data_utils.rand_int_id(1000000000, 9999999999))
create_database = (
'mysql -h localhost -e "CREATE DATABASE IF NOT EXISTS '
'{database}" '.format(database=database_name))
create_table = ('mysql -h localhost -e'
' "CREATE TABLE IF NOT EXISTS {database}.{table}'
' (data VARCHAR(100))" '.format(database=database_name,
table=table_name))
create_record = (
'mysql -h localhost -e "INSERT INTO {database}.{table} (data) '
'VALUES({data})" '.format(database=database_name,
table=table_name,
data=record_data))
get_record = (
'mysql -h localhost -e "SELECT * FROM {database}.{table} '
'WHERE data = \"{data}\"" '.format(database=database_name,
table=table_name,
data=record_data))
drop_db = "mysql -h localhost -e 'DROP DATABASE {database}'".format(
database=database_name)
# create db, table, insert data on one node
LOG.info('target node ip/hostname: "{0}" '.format(self.master_ip))
master_ssh_client = ssh.Client(self.master_ip,
self.node_user,
key_filename=self.node_key,
timeout=100)
self.verify(20, master_ssh_client.exec_command, 2,
'Database creation failed', 'create database',
create_database)
LOG.info('create database')
self.verify(20, master_ssh_client.exec_command, 3,
'Table creation failed', 'create table', create_table)
LOG.info('create table')
self.verify(20, master_ssh_client.exec_command, 4,
'Can not insert data in created table', 'data insertion',
create_record)
LOG.info('create data')
# Verify that data is replicated on other databases
for db_node in databases:
if db_node != self.master_ip:
client = ssh.Client(db_node,
self.node_user,
key_filename=self.node_key)
output = self.verify(
20, client.exec_command, 5,
'Can not get data from database node %s' % db_node,
'get_record', get_record)
self.verify_response_body(output,
record_data,
msg='Expected data missing',
failed_step='6')
# Drop created db
ssh_client = ssh.Client(self.master_ip,
self.node_user,
key_filename=self.node_key)
self.verify(20, ssh_client.exec_command, 7,
'Can not delete created database', 'database deletion',
drop_db)
self.master_ip = None
def test_state_of_galera_cluster(self):
"""Check galera environment state
Target Service: HA mysql
Scenario:
1. Detect there are online database nodes.
2. Ssh on each node containing database and request state of galera
node
3. For each node check cluster size
4. For each node check status is ready
5. For each node check that node is connected to cluster
Duration: 10 s.
"""
databases = self.verify(20,
self.get_database_nodes,
1, "Can not get database hostnames. Check that"
" at least one controller is operable",
"get database nodes",
self.controller_ip,
self.node_user,
key=self.node_key)
self.verify_response_body_not_equal(0, len(databases), self.no_db_msg,
1)
if len(databases) == 1:
self.skipTest(self.one_db_msg)
for db_node in databases:
command = "mysql -h localhost -e \"SHOW STATUS LIKE 'wsrep_%'\""
ssh_client = ssh.Client(db_node,
self.node_user,
key_filename=self.node_key,
timeout=100)
output = self.verify(
20, ssh_client.exec_command, 2,
"Verification of galera cluster node status failed",
'get status from galera node', command).splitlines()
LOG.debug('mysql output from node "{0}" is \n"{1}"'.format(
db_node, output))
mysql_vars = [
'wsrep_cluster_size', 'wsrep_ready', 'wsrep_connected'
]
result = self.get_variables_from_output(output, mysql_vars)
self.verify_response_body_content(result.get(
'wsrep_cluster_size', 0),
str(len(databases)),
msg='Cluster size on %s less '
'than databases count' % db_node,
failed_step='3')
self.verify_response_body_content(
result.get('wsrep_ready', 'OFF'),
'ON',
msg='wsrep_ready on %s is not ON' % db_node,
failed_step='4')
self.verify_response_body_content(
result.get('wsrep_connected', 'OFF'),
'ON',
msg='wsrep_connected on %s is not ON' % db_node,
failed_step='5')
