def test_jenkins_jobs_branch(local_salt_client):
config = utils.get_configuration()
expected_version = config['drivetrain_version'] or []
if not expected_version or expected_version == '':
pytest.skip("drivetrain_version is not defined. Skipping")
jenkins_password = get_password(local_salt_client, 'jenkins:client')
version_mismatch = []
server = join_to_jenkins(local_salt_client, 'admin', jenkins_password)
for job_instance in server.get_jobs():
job_name = job_instance.get('name')
job_config = server.get_job_config(job_name)
xml_data = minidom.parseString(job_config)
BranchSpec = xml_data.getElementsByTagName(
'hudson.plugins.git.BranchSpec')
#We use master branch for pipeline-library in case of 'testing,stable,nighlty' versions
if expected_version in ['testing', 'nightly', 'stable']:
expected_version = 'master'
if BranchSpec:
actual_version = BranchSpec[0].getElementsByTagName(
'name')[0].childNodes[0].data
if (actual_version != expected_version) and (job_name not in [
'cvp-func', 'cvp-ha', 'cvp-perf'
]):
version_mismatch.append("Job {0} has {1} branch."
"Expected {2}".format(
job_name, actual_version,
expected_version))
assert len(version_mismatch) == 0, \
'''Some DriveTrain jobs have version/branch mismatch:
{}'''.format(json.dumps(version_mismatch, indent=4))
def test_drivetrain_components_and_versions(local_salt_client):
config = utils.get_configuration()
version = config['drivetrain_version'] or []
if not version or version == '':
pytest.skip("drivetrain_version is not defined. Skipping")
salt_output = local_salt_client.cmd(
'I@gerrit:client',
'cmd.run',
['docker service ls'],
expr_form='compound')
not_found_services = ['gerrit_db', 'gerrit_server', 'jenkins_master',
'jenkins_slave01', 'jenkins_slave02',
'jenkins_slave03', 'ldap_admin', 'ldap_server']
version_mismatch = []
for line in salt_output[salt_output.keys()[0]].split('\n'):
for service in not_found_services:
if service in line:
not_found_services.remove(service)
if version != line.split()[4].split(':')[1]:
version_mismatch.append("{0}: expected "
"version is {1}, actual - {2}".format(service,version,
line.split()[4].split(':')[1]))
continue
assert len(not_found_services) == 0, \
'''Some DriveTrain components are not found:
{}'''.format(json.dumps(not_found_services, indent=4))
assert len(version_mismatch) == 0, \
'''Version mismatch found:
{}'''.format(json.dumps(version_mismatch, indent=4))
def test_drivetrain_jenkins_job(local_salt_client):
jenkins_password = get_password(local_salt_client,'jenkins:client')
server = join_to_jenkins(local_salt_client,'admin',jenkins_password)
#Getting Jenkins test job name from configuration
config = utils.get_configuration()
jenkins_test_job = config['jenkins_test_job']
if not jenkins_test_job or jenkins_test_job == '':
jenkins_test_job = 'git-mirror-downstream-mk-pipelines'
if server.get_job_name(jenkins_test_job):
next_build_num = server.get_job_info(jenkins_test_job)['nextBuildNumber']
#If this is first build number skip building check
if next_build_num != 1:
#Check that test job is not running at this moment,
#Otherwise skip the test
last_build_num = server.get_job_info(jenkins_test_job)['lastBuild'].get('number')
last_build_status = server.get_build_info(jenkins_test_job,last_build_num)['building']
if last_build_status:
pytest.skip("Test job {0} is already running").format(jenkins_test_job)
#This jenkins module doesn't work with build_job function without parameters
#Just send some fake parameters. All others will be used from default values
param_dict = {'foo':'bar'}
server.build_job(jenkins_test_job, param_dict)
timeout = 0
#Use job status True by default to exclude timeout between build job and start job.
job_status = True
while job_status and ( timeout < 180 ):
time.sleep(10)
timeout += 10
job_status = server.get_build_info(jenkins_test_job,next_build_num)['building']
job_result = server.get_build_info(jenkins_test_job,next_build_num)['result']
else:
pytest.skip("The job {0} was not found").format(test_job_name)
assert job_result == 'SUCCESS', \
'''Test job '{0}' build was not successfull or timeout is too small
'''.format(jenkins_test_job)
def test_checking_rabbitmq_cluster(local_salt_client):
# disable config for this test
# it may be reintroduced in future
config = utils.get_configuration()
# request pillar data from rmq nodes
rabbitmq_pillar_data = local_salt_client.cmd('rabbitmq:server',
'pillar.data',
['rabbitmq:cluster'],
expr_form='pillar')
# creating dictionary {node:cluster_size_for_the_node}
# with required cluster size for each node
control_dict = {}
required_cluster_size_dict = {}
# request actual data from rmq nodes
rabbit_actual_data = local_salt_client.cmd('rabbitmq:server',
'cmd.run',
['rabbitmqctl cluster_status'],
expr_form='pillar')
for node in rabbitmq_pillar_data:
if node in config.get('skipped_nodes'):
del rabbit_actual_data[node]
continue
cluster_size_from_the_node = len(
rabbitmq_pillar_data[node]['rabbitmq:cluster']['members'])
required_cluster_size_dict.update({node: cluster_size_from_the_node})
# find actual cluster size for each node
for node in rabbit_actual_data:
running_nodes_count = 0
# rabbitmqctl cluster_status output contains
# 3 * # of nodes 'rabbit@' entries + 1
running_nodes_count = (rabbit_actual_data[node].count('rabbit@') -
1) / 3
# update control dictionary with values
# {node:actual_cluster_size_for_node}
if required_cluster_size_dict[node] != running_nodes_count:
control_dict.update({node: running_nodes_count})
assert not len(control_dict), "Inconsistency found within cloud. " \
"RabbitMQ cluster is probably broken, " \
"the cluster size for each node " \
"should be: {} but the following " \
"nodes has other values: {}".format(
len(required_cluster_size_dict.keys()), control_dict)
def test_ntp_sync(local_salt_client):
testname = os.path.basename(__file__).split('.')[0]
active_nodes = utils.get_active_nodes(os.path.basename(__file__))
config = utils.get_configuration()
fail = {}
saltmaster_time = int(
local_salt_client.cmd('salt:master',
'cmd.run', ['date +%s'],
expr_form='pillar').values()[0])
nodes_time = local_salt_client.cmd(utils.list_to_target_string(
active_nodes, 'or'),
'cmd.run', ['date +%s'],
expr_form='compound')
diff = config.get(testname)["time_deviation"] or 30
for node, time in nodes_time.iteritems():
if (int(time) - saltmaster_time) > diff or \
(int(time) - saltmaster_time) < -diff:
fail[node] = time
assert not fail, 'SaltMaster time: {}\n' \
'Nodes with time mismatch:\n {}'.format(saltmaster_time,
fail)
def test_drivetrain_jenkins_job(local_salt_client):
jenkins_password = get_password(local_salt_client, 'jenkins:client')
server = join_to_jenkins(local_salt_client, 'admin', jenkins_password)
#Getting Jenkins test job name from configuration
config = utils.get_configuration()
jenkins_test_job = config['jenkins_test_job']
if not server.get_job_name(jenkins_test_job):
server.create_job(jenkins_test_job, jenkins.EMPTY_CONFIG_XML)
if server.get_job_name(jenkins_test_job):
next_build_num = server.get_job_info(
jenkins_test_job)['nextBuildNumber']
#If this is first build number skip building check
if next_build_num != 1:
#Check that test job is not running at this moment,
#Otherwise skip the test
last_build_num = server.get_job_info(
jenkins_test_job)['lastBuild'].get('number')
last_build_status = server.get_build_info(
jenkins_test_job, last_build_num)['building']
if last_build_status:
pytest.skip("Test job {0} is already running").format(
jenkins_test_job)
server.build_job(jenkins_test_job)
timeout = 0
#Use job status True by default to exclude timeout between build job and start job.
job_status = True
while job_status and (timeout < 180):
time.sleep(10)
timeout += 10
job_status = server.get_build_info(jenkins_test_job,
next_build_num)['building']
job_result = server.get_build_info(jenkins_test_job,
next_build_num)['result']
else:
pytest.skip("The job {0} was not found").format(test_job_name)
assert job_result == 'SUCCESS', \
'''Test job '{0}' build was not successfull or timeout is too small
'''.format(jenkins_test_job)
def test_jenkins_jobs_branch(local_salt_client):
config = utils.get_configuration()
expected_version = config['drivetrain_version'] or []
if not expected_version or expected_version == '':
pytest.skip("drivetrain_version is not defined. Skipping")
jenkins_password = local_salt_client.cmd(
'jenkins:client',
'pillar.get',
['_param:openldap_admin_password'],
expr_form='pillar').values()[0]
jenkins_port = local_salt_client.cmd(
'I@jenkins:client and not I@salt:master',
'pillar.get',
['_param:haproxy_jenkins_bind_port'],
expr_form='compound').values()[0]
jenkins_address = local_salt_client.cmd(
'I@jenkins:client and not I@salt:master',
'pillar.get',
['_param:haproxy_jenkins_bind_host'],
expr_form='compound').values()[0]
version_mismatch = []
jenkins_url = 'http://{0}:{1}'.format(jenkins_address,jenkins_port)
server = Jenkins(jenkins_url, username='******', password=jenkins_password)
for job_name, job_instance in server.get_jobs():
job_config = job_instance.get_config()
xml_data = minidom.parseString(job_config)
BranchSpec = xml_data.getElementsByTagName('hudson.plugins.git.BranchSpec')
if BranchSpec:
actual_version = BranchSpec[0].getElementsByTagName('name')[0].childNodes[0].data
if actual_version != expected_version and 'master' not in actual_version:
version_mismatch.append("Job {0} has {1} branch."
"Expected {2}".format(job_instance.name,
actual_version,
expected_version))
assert len(version_mismatch) == 0, \
'''Some DriveTrain jobs have version/branch mismatch:
{}'''.format(json.dumps(version_mismatch, indent=4))
def test_gerrit_repositories(local_salt_client):
missing_repos = []
config = utils.get_configuration()
gerrit_password = local_salt_client.cmd('I@gerrit:client',
'pillar.get',
['_param:openldap_admin_password'],
expr_form='compound').values()[0]
gerrit_port = local_salt_client.cmd('I@gerrit:client',
'pillar.get',
['gerrit:client:server:http_port'],
expr_form='compound').values()[0]
gerrit_address = local_salt_client.cmd('I@gerrit:client',
'pillar.get',
['gerrit:client:server:host'],
expr_form='compound').values()[0]
gerrit_protocol = local_salt_client.cmd('I@gerrit:client',
'pillar.get',
['gerrit:client:server:protocol'],
expr_form='compound').values()[0]
auth = HTTPBasicAuth('admin', gerrit_password)
rest = GerritRestAPI(url="{0}://{1}:{2}".format(gerrit_protocol,
gerrit_address,
gerrit_port),
auth=auth)
for repo in config['drivetrain_repos']:
repoHttp = repo.replace("/", "%2F")
try:
response = rest.get("/projects/{0}".format(repoHttp))
except requests.exceptions.HTTPError as e:
missing_repos.append("Repo {0} is missing".format(repo))
assert len(missing_repos) == 0, \
'''Some repositories in Gerrit are missing:
{}'''.format(json.dumps(missing_repos, indent=4))
def test_drivetrain_openldap(local_salt_client):
'''Create a test user 'DT_test_user' in openldap,
add the user to admin group, login using the user to Jenkins.
Add the user to devops group in Gerrit and then login to Gerrit,
using test_user credentials. Finally, delete the user from admin
group and openldap
'''
ldap_password = get_password(local_salt_client, 'openldap:client')
#Check that ldap_password is exists, otherwise skip test
if not ldap_password:
pytest.skip("Openldap service or openldap:client pillar \
are not found on this environment.")
ldap_port = local_salt_client.cmd(
'I@openldap:client and not I@salt:master',
'pillar.get', ['_param:haproxy_openldap_bind_port'],
expr_form='compound').values()[0]
ldap_address = local_salt_client.cmd(
'I@openldap:client and not I@salt:master',
'pillar.get', ['_param:haproxy_openldap_bind_host'],
expr_form='compound').values()[0]
ldap_dc = local_salt_client.cmd('openldap:client',
'pillar.get', ['_param:openldap_dn'],
expr_form='pillar').values()[0]
ldap_con_admin = local_salt_client.cmd(
'openldap:client',
'pillar.get', ['openldap:client:server:auth:user'],
expr_form='pillar').values()[0]
ldap_url = 'ldap://{0}:{1}'.format(ldap_address, ldap_port)
ldap_error = ''
ldap_result = ''
gerrit_result = ''
gerrit_error = ''
jenkins_error = ''
#Test user's CN
test_user_name = 'DT_test_user'
test_user = '******'.format(test_user_name, ldap_dc)
#Admins group CN
admin_gr_dn = 'cn=admins,ou=groups,{0}'.format(ldap_dc)
#List of attributes for test user
attrs = {}
attrs['objectclass'] = [
'organizationalRole', 'simpleSecurityObject', 'shadowAccount'
]
attrs['cn'] = test_user_name
attrs['uid'] = test_user_name
attrs['userPassword'] = '******'
attrs['description'] = 'Test user for CVP DT test'
searchFilter = 'cn={0}'.format(test_user_name)
#Get a test job name from config
config = utils.get_configuration()
jenkins_test_job = config['jenkins_test_job']
if not jenkins_test_job or jenkins_test_job == '':
jenkins_test_job = 'git-mirror-downstream-pipeline-library'
#Open connection to ldap and creating test user in admins group
try:
ldap_server = ldap.initialize(ldap_url)
ldap_server.simple_bind_s(ldap_con_admin, ldap_password)
ldif = modlist.addModlist(attrs)
ldap_server.add_s(test_user, ldif)
ldap_server.modify_s(
admin_gr_dn,
[(
ldap.MOD_ADD,
'memberUid',
[test_user_name],
)],
)
#Check search test user in LDAP
searchScope = ldap.SCOPE_SUBTREE
ldap_result = ldap_server.search_s(ldap_dc, searchScope, searchFilter)
except ldap.LDAPError, e:
ldap_error = e
def test_mtu(local_salt_client, nodes_in_group):
testname = os.path.basename(__file__).split('.')[0]
config = utils.get_configuration()
skipped_ifaces = config.get(testname)["skipped_ifaces"] or \
["bonding_masters", "lo", "veth", "tap", "cali"]
total = {}
network_info = local_salt_client.cmd("L@" + ','.join(nodes_in_group),
'cmd.run', ['ls /sys/class/net/'],
expr_form='compound')
kvm_nodes = local_salt_client.cmd('salt:control',
'test.ping',
expr_form='pillar').keys()
if len(network_info.keys()) < 2:
pytest.skip("Nothing to compare - only 1 node")
for node, ifaces_info in network_info.iteritems():
if node in kvm_nodes:
kvm_info = local_salt_client.cmd(node, 'cmd.run', [
"virsh list | "
"awk '{print $2}' | "
"xargs -n1 virsh domiflist | "
"grep -v br-pxe | grep br- | "
"awk '{print $1}'"
])
ifaces_info = kvm_info.get(node)
node_ifaces = ifaces_info.split('\n')
ifaces = {}
for iface in node_ifaces:
for skipped_iface in skipped_ifaces:
if skipped_iface in iface:
break
else:
iface_mtu = local_salt_client.cmd(
node, 'cmd.run',
['cat /sys/class/'
'net/{}/mtu'.format(iface)])
ifaces[iface] = iface_mtu.get(node)
total[node] = ifaces
nodes = []
mtu_data = []
my_set = set()
for node in total:
nodes.append(node)
my_set.update(total[node].keys())
for interf in my_set:
diff = []
row = []
for node in nodes:
if interf in total[node].keys():
diff.append(total[node][interf])
row.append("{}: {}".format(node, total[node][interf]))
else:
row.append("{}: No interface".format(node))
if diff.count(diff[0]) < len(nodes):
row.sort()
row.insert(0, interf)
mtu_data.append(row)
assert len(mtu_data) == 0, \
"Several problems found: {0}".format(
json.dumps(mtu_data, indent=4))
