def test_os_databases(self):
"""Check if amount of tables in databases is the same on each node
Target Service: HA mysql
Scenario:
1. Detect there are online database nodes.
2. Request list of tables for os databases on each node.
3. Check if amount of tables in databases is the same on each node
Duration: 10 s.
"""
LOG.info("'Test OS Databases' started")
dbs = ['nova', 'glance', 'keystone']
cmd = "mysql -h localhost -e 'SHOW TABLES FROM %(database)s'"
databases = self.verify(20, self.get_database_nodes,
1, "Can not get database hostnames. Check that"
" at least one controller is operable",
"get database nodes",
self.controller_ip,
self.node_user,
key=self.node_key)
if len(databases) == 1:
self.skipTest('There is only one database online. '
'Nothing to check')
for database in dbs:
LOG.info('Current database name is %s' % database)
temp_set = set()
for node in databases:
LOG.info('Current database node is %s' % node)
cmd1 = cmd % {'database': database}
LOG.info('Try to execute command %s' % cmd1)
tables = SSHClient(
node, self.node_user,
key_filename=self.node_key,
timeout=self.config.compute.ssh_timeout)
output = self.verify(40, tables.exec_command, 2,
'Can list tables',
'get amount of tables for each database',
cmd1)
tables = set(output.splitlines())
if len(temp_set) == 0:
temp_set = tables
self.verify_response_true(
len(tables.symmetric_difference(temp_set)) == 0,
"Step 3 failed: Tables in %s database are "
"different" % database)
del temp_set
def test_003_dns_resolution(self):
"""Check DNS resolution on compute node
Target component: OpenStack
Scenario:
1. Execute host 8.8.8.8 command from a compute node.
2. Check 8.8.8.8 host was successfully resolved
3. Check host google.com command from the compute node.
4. Check google.com host was successfully resolved.
Duration: 120 s.
Deployment tags: qemu | kvm, public_on_all_nodes | nova_network
"""
if not self.computes:
self.skipTest('There are no computes nodes')
dns = self.fuel_dns.spit(',') if self.fuel_dns else ['8.8.8.8']
ssh_client = SSHClient(self.computes[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
expected_output = "{0}.in-addr.arpa domain name pointer".format(dns[0])
cmd = "host {0}".format(dns[0])
output = self.verify(
100, self.retry_command, 1,
"'host' command failed. Looks like there is no "
"Internet connection on the computes node.", "'ping' command", 10,
5, ssh_client.exec_command, cmd)
LOG.debug(output)
self.verify_response_true(
expected_output in output, 'Step 2 failed: '
'DNS name for {0} host '
'cannot be resolved.'.format(dns[0]))
domain_name = output.split()[-1]
cmd = "host {0}".format(domain_name)
output = self.verify(
100, self.retry_command, 3, "'host' command failed. "
"DNS name cannot be resolved.", "'host' command", 10, 5,
ssh_client.exec_command, cmd)
LOG.debug(output)
self.verify_response_true(
'has address {0}'.format(dns[0]) in output, 'Step 4 failed: '
'DNS name cannot be resolved.')
def _run_ssh_cmd(self, cmd):
"""
Open SSH session with Controller and and execute command.
"""
if not self.host:
self.fail('Wrong test configuration: '
'"online_controllers" parameter is empty.')
try:
sshclient = SSHClient(self.host[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
return sshclient.exec_longrun_command(cmd)
except Exception:
LOG.debug(traceback.format_exc())
self.fail("%s command failed." % cmd)
def test_003_dns_resolution(self):
"""Check DNS resolution on compute node
Target component: OpenStack
Scenario:
1. Execute host 8.8.8.8 command from a compute node.
2. Check 8.8.8.8 host was successfully resolved
3. Check host google.com command from the compute node.
4. Check google.com host was successfully resolved.
Duration: 120 s.
Deployment tags: qemu | kvm, public_on_all_nodes | nova_network
"""
if not self.computes:
self.skipTest('There are no computes nodes')
ssh_client = SSHClient(self.computes[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
expected_output = "google"
cmd = "host 8.8.8.8"
output = self.verify(
100, self.retry_command, 1,
"'host' command failed. Looks like there is no "
"Internet connection on the computes node.", "'ping' command", 10,
5, ssh_client.exec_command, cmd)
LOG.debug(output)
self.verify_response_true(
expected_output in output, 'Step 2 failed: '
'DNS name for 8.8.8.8 host '
'cannot be resolved.')
expected_output = "google.com has address"
cmd = "host google.com"
output = self.verify(
100, self.retry_command, 3, "'host' command failed. "
"DNS name cannot be resolved.", "'host' command", 10, 5,
ssh_client.exec_command, cmd)
LOG.debug(output)
self.verify_response_true(
expected_output in output, 'Step 4 failed: '
'DNS name cannot be resolved.')
def get_database_nodes(cls, controller_ip, username, key):
# retrieve data from controller
ssh_client = SSHClient(controller_ip,
username,
key_filename=key,
timeout=100)
hiera_cmd = 'ruby -e \'require "hiera"; ' \
'puts Hiera.new().lookup("database_nodes", {}, {}).keys\''
database_nodes = ssh_client.exec_command(hiera_cmd)
database_nodes = database_nodes.splitlines()
# get online nodes
databases = []
for node in cls.config.compute.nodes:
hostname = node['hostname']
if hostname in database_nodes and node['online']:
databases.append(hostname)
return databases
def ping():
cmd = "ping -q -c1 -w10 %s" % ip_address
if self.host:
try:
ssh = SSHClient(self.host[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=timeout)
except Exception:
LOG.debug(traceback.format_exc())
return self.retry_command(retries[0], retries[1],
ssh.exec_command, cmd)
else:
self.fail('Wrong tests configurations, one from the next '
'parameters are empty controller_node_name or '
'controller_node_ip ')
def test_003_dns_resolution(self):
"""Check DNS resolution on a compute
Test DNS resolution on compute nodes.
Target component: OpenStack
Scenario:
1. Execute host 8.8.8.8 command from a compute node.
2. Check 8.8.8.8 host was successfully resolved
3. Check host google.com command from the compute node.
4. Check google.com host was successfully resolved.
Duration: 60 s.
"""
if not self.computes:
self.fail('Step 1 failed: There are no compute nodes')
ssh_client = SSHClient(self.computes[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
expected_output = "google"
cmd = "host 8.8.8.8"
output = self.verify(
50, ssh_client.exec_command, 1,
"'host' command failed. Looks like there is no "
"Internet connection on the compute node.", "'ping' command", cmd)
LOG.debug(output)
self.verify_response_true(
expected_output in output, 'Step 2 failed: '
'DNS name for 8.8.8.8 host '
'cannot be resolved.')
expected_output = "google.com has address"
cmd = "host google.com"
output = self.verify(
50, ssh_client.exec_command, 3, "'host' command failed. "
"DNS name cannot be resolved.", "'host' command", cmd)
LOG.debug(output)
self.verify_response_true(
expected_output in output, 'Step 4 failed: '
'DNS name cannot be resolved.')
def test_002_internet_connectivity_from_compute(self):
"""Check Internet connectivity from a compute
Test internet connections on compute nodes.
Target component: OpenStack
Scenario:
1. Execute ping 8.8.8.8 command from a compute node.
Duration: 40 s.
"""
if not self.computes:
self.fail('Step 1 failed: There are no compute nodes')
cmd = "ping 8.8.8.8 -c 1 -w 1"
ssh_client = SSHClient(self.computes[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
self.verify(
50, ssh_client.exec_command, 1,
"'ping' command failed. Looks like there is no "
"Internet connection on the compute node.", "'ping' command", cmd)
def get_database_nodes(cls, controller_ip, username, key):
if version.StrictVersion(cls.release_version)\
< version.StrictVersion('7.0'):
return cls.config.compute.online_controllers
# retrieve data from controller
ssh_client = SSHClient(controller_ip,
username,
key_filename=key,
timeout=100)
hiera_cmd = ('ruby -e \'require "hiera";'
'db = Hiera.new().lookup("database_nodes", {}, {}).keys;'
'if db != [] then puts db else puts "None" end\'')
database_nodes = ssh_client.exec_command(hiera_cmd)
# get online nodes
database_nodes = database_nodes.splitlines()
databases = []
for node in cls.config.compute.nodes:
hostname = node['hostname']
if hostname in database_nodes and node['online']:
databases.append(hostname)
return databases
def test_001_services_state(self):
"""Check that required services are running
Target component: OpenStack
Scenario:
1. Execute nova service-list command on a controller node.
2. Check there are no failed services (with down state).
Duration: 180 s.
"""
downstate = u'down'
cmd = 'source /root/openrc; nova service-list'
# FIXME(mattymo): Remove this after LP#1543625 is fixed in UCA
if self.fuel_repo_type == 'uca':
cmd = '{0} | egrep -v "metadata|osapi_compute"'.format(cmd)
if not self.controllers:
self.skipTest('Step 1 failed: there are no controller nodes.')
ssh_client = SSHClient(self.controllers[0],
self.usr,
self.pwd,
key_filename=self.key,
timeout=self.timeout)
output = self.verify(50, ssh_client.exec_command, 1,
"'nova service-list' command execution failed. ",
"'nova service-list' command execution", cmd)
LOG.debug(output)
try:
self.verify_response_true(
downstate not in output, 'Step 2 failed: Some nova services '
'have not been started.')
except Exception:
LOG.info("Will sleep for 120 seconds and try again")
LOG.debug(traceback.format_exc())
time.sleep(120)
self.verify_response_true(
downstate not in output, 'Step 2 failed: Some nova services '
'have not been started.')
def test_001_check_default_master_node_credential_usage(self):
"""Check usage of default credentials on master node
Target component: Configuration
Scenario:
1. Check user can not ssh on master node with default credentials.
Duration: 20 s.
Available since release: 2014.2-6.1
"""
ip = self.config.nailgun_host
ssh_client = SSHClient(ip,
self.config.master.master_node_ssh_user,
self.config.master.master_node_ssh_password,
timeout=self.config.master.ssh_timeout)
cmd = "date"
output = []
try:
output = ssh_client.exec_command(cmd)
LOG.debug(output)
except exceptions.SSHExecCommandFailed:
self.verify_response_true(
len(output) == 0, 'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
except exceptions.TimeoutException:
self.verify_response_true(
len(output) == 0, 'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
except exc.SSHException:
self.verify_response_true(
len(output) == 0, 'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
self.verify_response_true(
len(output) == 0, 'Step 1 failed: Default credentials for '
'ssh on master node were not changed')
def test_mysql_replication(self):
"""Check data replication over mysql
Target Service: HA mysql
Scenario:
1. Detect mysql node.
2. Create database on detected node
3. Create table in created database
4. Insert data to the created table
5. Get replicated data from each controller.
6. Verify that replicated data in the same from each controller
7. Drop created database
Duration: 100 s.
"""
# Find mysql master node
master_node_ip = []
cmd = 'mysql -e "SHOW SLAVE STATUS\G"'
LOG.info("Controllers nodes are %s" % self.controllers)
for controller_ip in self.controllers:
ssh_client = SSHClient(controller_ip,
self.controller_user,
key_filename=self.controller_key,
timeout=100)
output = self.verify(20, ssh_client.exec_command, 1,
'Mysql node detection failed',
'detect mysql node', cmd)
LOG.info('output is %s' % output)
if not output:
self.master_ip.append(controller_ip)
master_node_ip.append(controller_ip)
database_name = self.database
table_name = 'ost' + str(data_utils.rand_int_id(100, 999))
record_data = str(data_utils.rand_int_id(1000000000, 9999999999))
create_database = 'mysql -e "CREATE DATABASE IF NOT EXISTS %s"'\
% database_name
create_table = 'mysql -e "CREATE TABLE IF NOT EXISTS'\
' %(database)s.%(table)s'\
' (data VARCHAR(100))"'\
% {'database': database_name,
'table': table_name}
create_record = 'mysql -e "INSERT INTO %(database)s.%(table)s (data)'\
' VALUES(%(data)s)"'\
% {'database': database_name,
'table': table_name,
'data': record_data}
get_record = 'mysql -e "SELECT * FROM %(database)s.%(table)s '\
'WHERE data = \"%(data)s\""'\
% {'database': database_name,
'table': table_name,
'data': record_data}
# create db, table, insert data on master
LOG.info('master node ip %s' % master_node_ip[0])
master_ssh_client = SSHClient(master_node_ip[0],
self.controller_user,
key_filename=self.controller_key,
timeout=100)
self.verify(20, master_ssh_client.exec_command, 2,
'Database creation failed', 'create database',
create_database)
LOG.info('create database')
self.verify(20, master_ssh_client.exec_command, 3,
'Table creation failed', 'create table', create_table)
LOG.info('create table')
self.verify(20, master_ssh_client.exec_command, 4,
'Can not insert data in created table', 'data insertion',
create_record)
LOG.info('create data')
# Verify that data is replicated on other controllers
for controller in self.controllers:
if controller not in master_node_ip:
client = SSHClient(controller,
self.controller_user,
key_filename=self.controller_key)
output = self.verify(
20, client.exec_command, 5,
'Can not get data from controller %s' % controller,
'get_record', get_record)
self.verify_response_body(output,
record_data,
msg='Expected data missing',
failed_step='6')
# Drop created db
cmd = "mysql -e 'DROP DATABASE %s'" % self.database
ssh_client = SSHClient(master_node_ip[0],
self.controller_user,
key_filename=self.controller_key)
self.verify(20, ssh_client.exec_command, 7,
'Can not delete created database', 'database deletion',
cmd)
self.master_ip = []
def test_state_of_mysql_cluster(self):
"""Check mysql environment state
Target Service: HA mysql
Scenario:
1. Detect mysql master node.
2. Ssh on mysql-master node and request its status
3. Verify that position field is not empty
4. Ssh on mysql-slave nodes and request their statuses
5. Verify that Slave_IO_State is in appropriate state
6. Verify that Slave_IO_Running is in appropriate state
7. Verify that Slave_SQL_Running is in appropriate state
Duration: 100 s.
Deployment tags: RHEL
"""
if 'RHEL' in self.config.compute.deployment_os:
# Find mysql master node
master_node_ip = []
cmd = 'mysql -e "SHOW SLAVE STATUS\G"'
LOG.info("Controllers nodes are %s" % self.controllers)
for controller_ip in self.controllers:
ssh_client = SSHClient(controller_ip,
self.controller_user,
key_filename=self.controller_key,
timeout=100)
output = self.verify(20, ssh_client.exec_command, 1,
'Can not define master node',
'master mode detection', cmd)
LOG.info('output is %s' % output)
if not output:
self.master_ip.append(controller_ip)
master_node_ip.append(controller_ip)
# ssh on master node and check status
check_master_state_cmd = 'mysql -e "SHOW MASTER STATUS\G"'
ssh_client = SSHClient(self.master_ip[0],
self.controller_user,
key_filename=self.controller_key,
timeout=100)
output = self.verify(
20, ssh_client.exec_command, 2,
'Can not execute "SHOW MASTER STATUS" '
'command', 'check master status',
check_master_state_cmd).splitlines()[1:]
LOG.info('master output is %s' % output)
res = [data.strip().split(':') for data in output]
master_dict = dict((k, v) for (k, v) in res)
self.verify_response_body_not_equal(
master_dict['Position'],
'',
msg='Position field is empty. Master is offline',
failed_step='3')
# ssh on slave node and check it status
check_slave_state_cmd = 'mysql -e "SHOW SLAVE STATUS\G"'
for controller in self.controllers:
if controller not in self.master_ip:
client = SSHClient(controller,
self.controller_user,
key_filename=self.controller_key)
output = self.verify(
20, client.exec_command, 4,
'Failed to get slave status', 'get slave status',
check_slave_state_cmd).splitlines()[1:19]
LOG.info("slave output is %s" % output)
res = [data.strip().split(':') for data in output]
slave_dict = dict((k, v) for (k, v) in res)
self.verify_response_body(
slave_dict['Slave_IO_State'],
' Waiting for master to send event',
msg='Slave IO state is incorrect ',
failed_step='5')
self.verify_response_body(
slave_dict['Slave_IO_Running'],
' Yes',
msg='Slave_IO_Running state is incorrect',
failed_step='6')
self.verify_response_body(
slave_dict['Slave_SQL_Running'],
' Yes',
msg='Slave_SQL_Running state is incorrect',
failed_step='7')
else:
self.skipTest("There is no RHEL deployment")
def test_state_of_galera_cluster(self):
"""Check galera environment state
Target Service: HA mysql
Scenario:
1. Detect there are online database nodes.
2. Ssh on each node containing database and request state of galera
node
3. For each node check cluster size
4. For each node check status is ready
5. For each node check that node is connected to cluster
Duration: 10 s.
"""
databases = self.verify(20,
self.get_database_nodes,
1, "Can not get database hostnames. Check that"
" at least one controller is operable",
"get database nodes",
self.controller_ip,
self.node_user,
key=self.node_key)
self.verify_response_body_not_equal(0, len(databases), self.no_db_msg,
1)
if len(databases) == 1:
self.skipTest(self.one_db_msg)
for db_node in databases:
command = "mysql -h localhost -e \"SHOW STATUS LIKE 'wsrep_%'\""
ssh_client = SSHClient(db_node,
self.node_user,
key_filename=self.node_key,
timeout=100)
output = self.verify(
20, ssh_client.exec_command, 2,
"Verification of galera cluster node status failed",
'get status from galera node', command).splitlines()
LOG.debug('mysql output from node "{0}" is \n"{1}"'.format(
db_node, output))
mysql_vars = [
'wsrep_cluster_size', 'wsrep_ready', 'wsrep_connected'
]
result = self.get_variables_from_output(output, mysql_vars)
self.verify_response_body_content(result.get(
'wsrep_cluster_size', 0),
str(len(databases)),
msg='Cluster size on %s less '
'than databases count' % db_node,
failed_step='3')
self.verify_response_body_content(
result.get('wsrep_ready', 'OFF'),
'ON',
msg='wsrep_ready on %s is not ON' % db_node,
failed_step='4')
self.verify_response_body_content(
result.get('wsrep_connected', 'OFF'),
'ON',
msg='wsrep_connected on %s is not ON' % db_node,
failed_step='5')
def test_mysql_replication(self):
"""Check data replication over mysql
Target Service: HA mysql
Scenario:
1. Check that mysql is running on all controller or database nodes.
2. Create database on one node.
3. Create table in created database
4. Insert data to the created table
5. Get replicated data from each database node.
6. Verify that replicated data in the same from each database
7. Drop created database
Duration: 10 s.
"""
LOG.info("'Test MySQL replication' started")
databases = self.verify(20,
self.get_database_nodes,
1, "Can not get database hostnames. Check that"
" at least one controller is operable",
"get database nodes",
self.controller_ip,
self.node_user,
key=self.node_key)
if len(databases) == 1:
self.skipTest('There is only one database online. '
'Nothing to check')
LOG.info("Database nodes are " + ", ".join(databases))
self.master_ip = databases[0]
# check that mysql is running on all hosts
cmd = 'mysql -h localhost -e "" '
for db_node in databases:
ssh_client = SSHClient(db_node,
self.node_user,
key_filename=self.node_key,
timeout=100)
self.verify(
20, ssh_client.exec_command, 1, 'Can not connect to mysql. '
'Please check that mysql is running and there '
'is connectivity by management network', 'detect mysql node',
cmd)
database_name = self.database
table_name = 'ost' + str(data_utils.rand_int_id(100, 999))
record_data = str(data_utils.rand_int_id(1000000000, 9999999999))
create_database = (
'mysql -h localhost -e "CREATE DATABASE IF NOT EXISTS '
'{database}" '.format(database=database_name))
create_table = ('mysql -h localhost -e'
' "CREATE TABLE IF NOT EXISTS {database}.{table}'
' (data VARCHAR(100))" '.format(database=database_name,
table=table_name))
create_record = (
'mysql -h localhost -e "INSERT INTO {database}.{table} (data) '
'VALUES({data})" '.format(database=database_name,
table=table_name,
data=record_data))
get_record = (
'mysql -h localhost -e "SELECT * FROM {database}.{table} '
'WHERE data = \"{data}\"" '.format(database=database_name,
table=table_name,
data=record_data))
drop_db = "mysql -h localhost -e 'DROP DATABASE {database}'".format(
database=database_name)
# create db, table, insert data on one node
LOG.info('target node ip/hostname: "{0}" '.format(self.master_ip))
master_ssh_client = SSHClient(self.master_ip,
self.node_user,
key_filename=self.node_key,
timeout=100)
self.verify(20, master_ssh_client.exec_command, 2,
'Database creation failed', 'create database',
create_database)
LOG.info('create database')
self.verify(20, master_ssh_client.exec_command, 3,
'Table creation failed', 'create table', create_table)
LOG.info('create table')
self.verify(20, master_ssh_client.exec_command, 4,
'Can not insert data in created table', 'data insertion',
create_record)
LOG.info('create data')
# Verify that data is replicated on other databases
for db_node in databases:
if db_node != self.master_ip:
client = SSHClient(db_node,
self.node_user,
key_filename=self.node_key)
output = self.verify(
20, client.exec_command, 5,
'Can not get data from database node %s' % db_node,
'get_record', get_record)
self.verify_response_body(output,
record_data,
msg='Expected data missing',
failed_step='6')
# Drop created db
ssh_client = SSHClient(self.master_ip,
self.node_user,
key_filename=self.node_key)
self.verify(20, ssh_client.exec_command, 7,
'Can not delete created database', 'database deletion',
drop_db)
self.master_ip = None