def test_check_package_versions(local_salt_client, group):
config = utils.get_configuration(__file__)
output = local_salt_client.cmd(group, 'lowpkg.list_pkgs', expr_form='pcre')
if len(output.keys()) < 2:
pytest.skip("Nothing to compare - only 1 node")
nodes = []
pkts_data = []
my_set = set()
for node in output:
nodes.append(node)
my_set.update(output[node].keys())
for deb in my_set:
diff = []
row = []
for node in nodes:
if deb in output[node].keys():
diff.append(output[node][deb])
row.append("{}: {}".format(node, output[node][deb]))
else:
row.append("{}: No package".format(node))
if diff.count(diff[0]) < len(nodes):
row.sort()
row.insert(0, deb)
pkts_data.append(row)
assert len(pkts_data) <= 1, \
"Several problems found for {0} group: {1}".format(
group, json.dumps(pkts_data, indent=4))
def test_speed_glance(create_image, glance_client):
"""
Simplified Performance Tests Download / upload lance
1. Step download image
2. Step upload image
"""
config = utils.get_configuration(__file__)
image = glance_client.images.create(name="test_image",
disk_format='iso',
container_format='bare')
start_time = time.time()
glance_client.images.upload(image.id,
image_data=open("/tmp/image_mk_framework.dd",
'rb'))
end_time = time.time()
speed_upload = int(config['size_image_mb']) / (end_time - start_time)
start_time = time.time()
with open("/tmp/image_mk_framework.download", 'wb') as image_file:
for item in glance_client.images.data(image.id):
image_file.write(item)
end_time = time.time()
speed_download = int(config['size_image_mb']) / (end_time - start_time)
glance_client.images.delete(image.id)
print "++++++++++++++++++++++++++++++++++++++++"
print 'upload - {} Mb/s'.format(speed_upload)
print 'download - {} Mb/s'.format(speed_download)
def test_check_services(local_salt_client, group):
config = utils.get_configuration(__file__)
output = local_salt_client.cmd(group, 'service.get_all', expr_form='pcre')
if len(output.keys()) < 2:
pytest.skip("Nothing to compare - only 1 node")
nodes = []
pkts_data = []
my_set = set()
for node in output:
nodes.append(node)
my_set.update(output[node])
for srv in my_set:
diff = []
row = []
for node in nodes:
if srv in output[node]:
diff.append(srv)
row.append("{}: +".format(node))
else:
row.append("{}: No service".format(node))
if diff.count(diff[0]) < len(nodes):
row.sort()
row.insert(0, srv)
pkts_data.append(row)
assert len(pkts_data) <= 1, \
"Several problems found for {0} group: {1}".format(
group, json.dumps(pkts_data, indent=4))
def test_check_ceph_osd(local_salt_client):
config = utils.get_configuration(__file__)
osd_fail = \
local_salt_client.cmd(config["ceph_osd_probe_node"][0], 'cmd.run',
['ceph osd tree | grep down'])
assert not osd_fail.values()[0], \
"Some osds are in down state or ceph is not found".format(
osd_fail.values()[0])
def vm_kp():
config = utils.get_configuration(__file__)
mk_nova = nova.Nova()
vm = mk_nova.create_vm('qa-framework-{}'.format(random.randint(1, 100)),
key_name=config['key_name'])
mk_nova.wait_for_vm_status_is_active(vm.id)
yield vm
vm.delete()
def test_mtu(local_salt_client, group):
config = utils.get_configuration(__file__)
skipped_ifaces = config["skipped_ifaces"]
expected_mtu = config["expected_mtu"]
total = {}
failed_ifaces = {}
network_info = local_salt_client.cmd(group, 'cmd.run',
['sudo ls /sys/class/net/'])
for node, ifaces_info in network_info.iteritems():
if 'kvm' in node:
kvm_info = local_salt_client.cmd(node, 'cmd.run',
["virsh list | grep jse2 | "
"awk '{print $2}' | "
"xargs -n1 virsh domiflist | "
"grep -v br-pxe | grep br- | "
"awk '{print $1}'"])
ifaces_info = kvm_info.get(node)
group_name = node.split('-')[0]
node_ifaces = ifaces_info.split('\n')
if group_name not in expected_mtu:
continue
else:
ifaces = {}
for iface in node_ifaces:
if iface in skipped_ifaces:
continue
iface_mtu = local_salt_client.cmd(node, 'cmd.run',
['cat /sys/class/'
'net/{}/mtu'.format(iface)])
ifaces[iface] = iface_mtu.get(node)
total[node] = ifaces
for node in total:
ifaces = total.get(node)
group_name = node.split('-')[0]
for iface in ifaces:
if group_name not in expected_mtu:
continue
else:
group = expected_mtu.get(group_name)
gauge = group.get(iface)
mtu = ifaces.get(iface)
if iface not in expected_mtu:
continue
elif int(mtu) != int(gauge):
failed_ifaces[node].append(iface)
assert not failed_ifaces, "Nodes with " \
"iface mismatch: {}".format(failed_ifaces)
def create_image():
config = utils.get_configuration(__file__)
line = 'dd if=/dev/zero of=/tmp/image_mk_framework.dd ' \
'bs=1M count={}'.format(config['size_image_mb'])
subprocess.call(line.split())
yield
# teardown
subprocess.call('rm /tmp/image_mk_framework.dd'.split())
subprocess.call('rm /tmp/image_mk_framework.download'.split())
def create_vm(self, name, key_name=None):
config = utils.get_configuration(__file__)
image_id = config['image_ref']
flavor_id = config['flavor_ref']
net_id = config['network_ref']
nics = [{"net-id": net_id, "v4-fixed-ip": ''}]
vm = self.client.servers.create(
name, image_id, flavor_id, nics=nics, key_name=key_name)
return vm
def __init__(self):
config = utils.get_configuration(__file__)
# TODO(den) openstack catalog list
version = '2.1'
client = nv_client.Client(
version,
config['admin_username'],
config['admin_password'],
config['admin_project_id'],
config['url'],
service_type="compute",
endpoint_type=config['endpoint_type'])
self.client = client
def create_vm(nova_client):
config = utils.get_configuration(__file__)
image_id = config['image_ref']
flavor_id = config['flavor_ref']
net_id = config['network_ref']
nics = [{"net-id": net_id, "v4-fixed-ip": ''}]
vm = nova_client.servers.create('test_migration',
image_id,
flavor_id,
nics=nics)
return vm
def test_dd(vm_kp, floating_ip, request, global_results, count, block_size):
config = utils.get_configuration(__file__)
vm_kp.add_floating_ip(floating_ip)
time.sleep(60)
ssh_client = ssh.Node('vm',
floating_ip.ip,
'ubuntu',
pub_key=config['key_path'])
ssh_client.run('rm test.dat')
ssh_stdout = ssh_client.run('dd if=/dev/zero of=test.dat \
oflag=direct bs={0} count={1}'.format(
count, block_size))
global_results.update(
request.node.name,
ssh_stdout.split(" ")[-2] + " " + ssh_stdout.split(" ")[-1])
def glance_client():
config = utils.get_configuration(__file__)
auth = v3.Password(auth_url=config['url_v3'],
username=config['admin_username'],
password=config['admin_password'],
project_id=config['admin_project_id'],
user_domain_id='default',
project_domain_id='default')
sess = session.Session(auth=auth, verify=False)
endpoint = auth.get_endpoint(session=sess,
service_type='image',
interface='internal')
client = gl_client.Client(config['glance_version'], endpoint, session=sess)
return client
def test_migration_for_all_nodes(nova_client, create_vm):
config = utils.get_configuration(__file__)
zone = config['availability_zone']
list_hosts = [host.host for host in nova_client.hosts.list(zone)]
utils.wait_for_vm_status_is_active(nova_client, create_vm.id)
vm_host = nova_client.servers.find(id=create_vm.id)
# replacing the current host to last place
list_hosts.remove(vm_host._info['OS-EXT-SRV-ATTR:host'])
list_hosts.append(vm_host._info['OS-EXT-SRV-ATTR:host'])
for host in list_hosts:
create_vm.live_migrate(host=host)
checking_vm_host(nova_client, create_vm.id, host)
create_vm.delete()
def test_ceph_haproxy(local_salt_client):
config = utils.get_configuration(__file__)
fail = {}
for monitor in config["ceph_monitors"]:
monitor_info = local_salt_client.cmd(monitor, 'cmd.run', [
"echo 'show stat' | nc -U "
"/var/run/haproxy/admin.sock | "
"grep ceph_mon_radosgw_cluster"
])
for name, info in monitor_info.iteritems():
if "OPEN" and "UP" in info:
continue
else:
fail[name] = info
assert not fail, "Failed monitors: {}".format(fail)
def test_check_default_gateways(local_salt_client, group):
config = utils.get_configuration(__file__)
netstat_info = local_salt_client.cmd(group,
'cmd.run', ['ip r | sed -n 1p'],
expr_form='pcre')
gateways = {}
nodes = netstat_info.keys()
for node in nodes:
if netstat_info[node] not in gateways:
gateways[netstat_info[node]] = [node]
else:
gateways[netstat_info[node]].append(node)
assert len(gateways.keys()) == 1, \
"There were found few gateways within group {group}: {gw}".format(
group=group,
gw=json.dumps(gateways, indent=4)
)
def test_ntp_sync(local_salt_client, node):
config = utils.get_configuration(__file__)
fail = {}
saltmaster_time = int(
local_salt_client.cmd(os.uname()[1] + '*', 'cmd.run',
['date +%s']).values()[0])
nodes_time = local_salt_client.cmd(node,
'cmd.run', ['date +%s'],
expr_form='pcre')
for node, time in nodes_time.iteritems():
if (int(time) - saltmaster_time) > config["time_deviation"] or \
(int(time) - saltmaster_time) < -config["time_deviation"]:
fail[node] = time
assert not fail, 'SaltMaster time: {}\n' \
'Nodes with time mismatch:\n {}'.format(saltmaster_time,
fail)
def test_check_module_versions(local_salt_client, group):
config = utils.get_configuration(__file__)
pre_check = local_salt_client.cmd(group,
'cmd.run',
['dpkg -l | grep "python-pip "'],
expr_form='pcre')
if pre_check.values().count('') > 0:
pytest.skip("pip is not installed on one or more nodes")
if len(pre_check.keys()) < 2:
pytest.skip("Nothing to compare - only 1 node")
output = local_salt_client.cmd(group, 'pip.freeze', expr_form='pcre')
nodes = []
pkts_data = []
my_set = set()
for node in output:
nodes.append(node)
my_set.update([x.split("=")[0] for x in output[node]])
output[node] = dict([x.split("==") for x in output[node]])
for deb in my_set:
diff = []
row = []
for node in nodes:
if deb in output[node].keys():
diff.append(output[node][deb])
row.append("{}: {}".format(node, output[node][deb]))
else:
row.append("{}: No module".format(node))
if diff.count(diff[0]) < len(nodes):
row.sort()
row.insert(0, deb)
pkts_data.append(row)
assert len(pkts_data) <= 1, \
"Several problems found for {0} group: {1}".format(
group, json.dumps(pkts_data, indent=4))
def test_mtu(local_salt_client, group):
config = utils.get_configuration(__file__)
skipped_ifaces = config["skipped_ifaces"]
total = {}
network_info = local_salt_client.cmd(
group,
'cmd.run', [
'ls /sys/class/net/ | grep -v "veth" | grep -v "tap" | grep -v "cali"'
],
expr_form='pcre')
kvm_nodes = local_salt_client.cmd('salt:control',
'test.ping',
expr_form='pillar').keys()
if len(network_info.keys()) < 2:
pytest.skip("Nothing to compare - only 1 node")
for node, ifaces_info in network_info.iteritems():
if node in kvm_nodes:
kvm_info = local_salt_client.cmd(node, 'cmd.run', [
"virsh list | "
"awk '{print $2}' | "
"xargs -n1 virsh domiflist | "
"grep -v br-pxe | grep br- | "
"awk '{print $1}'"
])
ifaces_info = kvm_info.get(node)
node_ifaces = ifaces_info.split('\n')
ifaces = {}
for iface in node_ifaces:
if iface in skipped_ifaces:
continue
iface_mtu = local_salt_client.cmd(
node, 'cmd.run',
['cat /sys/class/'
'net/{}/mtu'.format(iface)])
ifaces[iface] = iface_mtu.get(node)
total[node] = ifaces
nodes = []
mtu_data = []
my_set = set()
for node in total:
nodes.append(node)
my_set.update(total[node].keys())
for interf in my_set:
diff = []
row = []
for node in nodes:
if interf in total[node].keys():
diff.append(total[node][interf])
row.append("{}: {}".format(node, total[node][interf]))
else:
row.append("{}: No interface".format(node))
if diff.count(diff[0]) < len(nodes):
row.sort()
row.insert(0, interf)
mtu_data.append(row)
assert len(mtu_data) == 0, \
"Several problems found for {0} group: {1}".format(
group, json.dumps(mtu_data, indent=4))
import pytest
from mk_verificator import utils
from collections import Counter
@pytest.mark.parametrize("group",
utils.get_groups(utils.get_configuration(__file__)))
def test_single_vip(local_salt_client, group):
local_salt_client.cmd(group, 'saltutil.sync_all')
nodes_list = local_salt_client.cmd(group, 'grains.item', ['ipv4'])
ipv4_list = []
for node in nodes_list:
ipv4_list.extend(nodes_list.get(node).get('ipv4'))
cnt = Counter(ipv4_list)
for ip in cnt:
if ip == '127.0.0.1':
continue
elif cnt[ip] > 1:
assert "VIP IP duplicate found " \
"in group {}\n{}".format(group, ipv4_list)
import pytest
import os
from mk_verificator import utils
@pytest.mark.parametrize("node",
utils.get_active_nodes(
utils.get_configuration(__file__)))
def test_ntp_sync(local_salt_client, node):
config = utils.get_configuration(__file__)
fail = {}
saltmaster_time = int(
local_salt_client.cmd(os.uname()[1] + '*', 'cmd.run',
['date +%s']).values()[0])
nodes_time = local_salt_client.cmd(node,
'cmd.run', ['date +%s'],
expr_form='pcre')
for node, time in nodes_time.iteritems():
if (int(time) - saltmaster_time) > config["time_deviation"] or \
(int(time) - saltmaster_time) < -config["time_deviation"]:
fail[node] = time
assert not fail, 'SaltMaster time: {}\n' \
'Nodes with time mismatch:\n {}'.format(saltmaster_time,
fail)
