def log_analyzer(job_id):
"""
Analyse the log file got from polarion
:param job_id: polarion job id
"""
ret, output = subprocess.getstatusoutput(
f'curl -k '
f'-u {args.username}:{args.password} '
f'-X GET {args.url}-log?jobId={job_id} > {args.log_file}')
if ret == 0:
ret, output = subprocess.getstatusoutput(f'cat {args.log_file}')
output = output.replace('"', '\"').replace(''', '\'')
log = json.loads(output[output.rfind('Message Content:') +
17:output.rfind('}') + 1],
strict=False)
case_pass_num = 0
case_fail_num = 0
log_url = log['log-url']
case_list = log['import-testcases']
for i in range(len(case_list)):
case_status = case_list[i]['status']
if case_status == 'passed':
case_pass_num += 1
else:
case_fail_num += 1
case_total_num = case_pass_num + case_fail_num
logger.info(f'Total uploading case number: {case_total_num}\n'
f'Passed uploading case number: {case_pass_num}\n'
f'Failed uploading case number: {case_fail_num}\n'
f'Log URL: {log_url}')
if case_fail_num > 0:
raise FailException('Failed to upload all test cases to polarion')
else:
raise FailException('Failed to get the polarion job log')
def associate(self, hypervisor, guest):
"""
Check the hypervisor is associated with guest on web.
:param guest: guest name
:param hypervisor: hypervisor host name/uuid/hwuuid
"""
host_id = self.host_id(host=hypervisor)
guest_id = self.host_id(host=guest)
if host_id and guest_id:
# Find the guest in hypervisor page
ret, output = request_get(url=f'{self.api}/api/v2/hosts/{host_id}',
auth=self.auth)
if guest.lower() in str(output):
logger.info(
'Succeeded to find the associated guest in hypervisor page')
else:
raise FailException(
'Failed to find the associated guest in hypervisor page')
# Find the hypervisor in guest page
ret, output = request_get(url=f'{self.api}/api/v2/hosts/{guest_id}',
auth=self.auth)
if hypervisor.lower() in str(output):
logger.info(
'Succeeded to find the associated hypervisor in guest page')
else:
raise FailException(
'Failed to find the associated hypervisor in guest page')
def satellite_manifest_upload(ssh, url, admin_username, admin_password):
"""
Upload manifest to satellite by hammer command.
:param ssh: ssh access to satellite host.
:param url: manifest url
:param admin_username: username of admin account.
:param admin_password: password of admin account.
"""
path = "/tmp/manifest"
ssh.runcmd(f'rm -rf {path}; mkdir -p {path}')
ssh.runcmd(f'wget {url} -P {path}')
ret, output = ssh.runcmd(f'ls {path}')
if output:
filename = f'{path}/{output.strip()}'
else:
raise FailException('No manifest file found')
ret, _ = ssh.runcmd(f'hammer -u {admin_username} -p {admin_password} '
f'subscription upload '
f'--organization-label Default_Organization '
f'--file {filename}')
if ret != 0:
raise FailException('Failed to upload manifest for satellite')
ret, _ = ssh.runcmd(f'hammer -u {admin_username} -p {admin_password} '
f'subscription refresh-manifest '
f'--organization="Default Organization"')
if ret != 0:
raise FailException('Failed to refresh satellite manifest')
def grub_update(ssh, ks_url, vmlinuz_url, initrd_url, repo_url):
"""
Update grub menuentry
:param ssh: ssh access to the host
:param ks_url: url of ks file
:param vmlinuz_url: url of compose vmlinuz file
:param initrd_url: url of compose initrd file
:param repo_url: url of compose repo
"""
logger.info(f'-----{ks_url}')
logger.info(f'-----{vmlinuz_url}')
logger.info(f'-----{initrd_url}')
logger.info(f'-----{repo_url}')
if (not base.url_validation(ks_url) or not base.url_validation(vmlinuz_url)
or not base.url_validation(initrd_url)
or not base.url_validation(repo_url)):
raise FailException('The necessary urls are not available')
menu_title = 'rhel-reinstall'
vmlinuz_name = 'vmlinuz-reinstall'
initrd_name = 'initrd-reinstall.img'
ssh.runcmd(f'rm -f /boot/{vmlinuz_name};'
f'curl -L {vmlinuz_url} -o /boot/{vmlinuz_name};'
f'sync')
ssh.runcmd(f'rm -f /boot/{initrd_name};'
f'curl -L {initrd_url} -o /boot/{initrd_name};'
f'sync')
cmd = (
'cat <<EOF > /etc/grub.d/40_custom\n'
'#!/bin/sh\n'
'exec tail -n +3 \$0\n'
"menuentry '%s' --class red --class gnu-linux --class gnu --class os {\n"
'load_video\n'
'set gfxpayload=keep\n'
'insmod gzio\n'
'insmod part_msdos\n'
'insmod xfs\n'
'set root="hd0,msdos1"\n'
'linux16 /%s ksdevice=bootif ip=dhcp ks=%s repo=%s quiet LANG=en_US.UTF-8 acpi=off\n'
'initrd16 /%s\n'
'}\n'
'EOF') % (menu_title, vmlinuz_name, ks_url, repo_url, initrd_name)
ret1, _ = ssh.runcmd(cmd)
ret2, _ = ssh.runcmd('grub2-mkconfig -o /boot/grub2/grub.cfg')
ret3, _ = ssh.runcmd(
f'grub2-set-default "{menu_title}"; grub2-editenv list')
if ret1 != 0 or ret2 != 0 or ret3 != 0:
raise FailException('Failed to update grub file.')
time.sleep(60)
def attach(self, pool=None, quantity=None):
"""
Attach subscription by Pool ID or --auto.
:param pool: Pool ID, attach by --auto when pool=None
:param quantity: subscription number to attach, default is auto.
:return: tty output.
"""
cmd = 'subscription-manager attach '
if pool:
cmd += f'--pool={pool} '
if quantity:
cmd += f'--quantity={quantity}'
if not pool:
cmd += f'--auto '
self.refresh()
ret, output = self.ssh.runcmd(cmd)
if ret == 0:
logger.info(f'Succeeded to attach subscription for {self.host}')
return output.strip()
if '--auto' in cmd and 'Unable to find available' in output:
logger.warning(
f'Failed to attach subscription by auto for {self.host}.')
return output.strip()
if 'Multi-entitlement not supported' in output:
logger.warning(output)
return output.strip()
else:
raise FailException(f'Failed to attach subscription for {self.host}')
def install_rhel_by_beaker(args):
"""
Install rhel os by submitting job to beaker with required arguments.
Please refer to the utils/README for usage.
"""
job_name = f'virtwho-{args.rhel_compose}'
ssh_client = SSHConnect(
host=config.beaker.client,
user=config.beaker.client_username,
pwd=config.beaker.client_password
)
beaker_client_kinit(
ssh_client,
config.beaker.keytab,
config.beaker.principal
)
job_id = beaker_job_submit(
ssh_client,
job_name,
args.rhel_compose,
args.arch,
args.variant,
args.job_group,
args.host,
args.host_type,
args.host_require,
)
while beaker_job_status(ssh_client, job_name, job_id):
time.sleep(60)
host = beaker_job_result(ssh_client, job_name, job_id)
if host:
logger.info(f'Succeeded to install {args.rhel_compose} by beaker '
f'({host})')
return host
raise FailException('Failed to install {args.rhel_compose} by beaker')
def satellite_repo_enable_dogfood(ssh,
rhel_ver,
sat_ver,
repo_type='satellite'):
"""
Enable the required repos for installing satellite that is still
in development.
:param ssh: ssh access to satellite host.
:param sat_ver: satellite version, such as 6.8, 6.9.
:param rhel_ver: rhel version, such as 6, 7, 8.
:param repo_type: satellite, capsule or satellite-tools.
:return: True or raise Fail.
"""
maintenance_pool = '8a88800f5ca45116015cc807610319ed'
dogfood = config.satellite.dogfood
org = 'Sat6-CI'
ssh.runcmd('subscription-manager unregister;' 'subscription-manager clean')
ssh.runcmd('rpm -qa |'
'grep katello-ca-consumer |'
'xargs rpm -e |'
'sort')
ssh.runcmd(f'yum -y localinstall {dogfood}')
ret, _ = ssh.runcmd(
f'subscription-manager register '
f'--org {org} '
f'--activationkey "{repo_type}-{sat_ver}-qa-rhel{rhel_ver}"')
if ret == 0:
ssh.runcmd(f'subscription-manager attach '
f'--pool {maintenance_pool}')
return True
raise FailException('Failed to enable dogfood repo.')
def virtwho_install(ssh, url=None):
"""
Install virt-who package, default is from repository,
or gating msg, or brew url.
:param ssh: ssh access of virt-who host
:param url: url link of virt-who package from brew
"""
rhel_ver = base.rhel_version(ssh)
cmd = ('rm -rf /var/lib/rpm/__db*;'
'mv /var/lib/rpm /var/lib/rpm.old;'
'rpm --initdb;'
'rm -rf /var/lib/rpm;'
'mv /var/lib/rpm.old /var/lib/rpm;'
'rm -rf /var/lib/yum/history/*.sqlite;'
'rpm -v --rebuilddb')
if rhel_ver == '6':
cmd = 'dbus-uuidgen > /var/lib/dbus/machine-id'
if rhel_ver == '8':
cmd = 'localectl set-locale en_US.utf8; source /etc/profile.d/lang.sh'
_, _ = ssh.runcmd(cmd)
if url:
virtwho_install_by_url(ssh, url)
else:
ssh.runcmd('yum remove -y virt-who;' 'yum install -y virt-who')
_, output = ssh.runcmd('rpm -qa virt-who')
if 'virt-who' not in output:
raise FailException('Failed to install virt-who package')
logger.info(f'Succeeded to install {output.strip()}')
return output.strip()
def run_start(self, cli=None, wait=None):
"""
Start/loop to run virt-who and analyze the result mainly by the
rhsm log.
:param cli: the command to run virt-who, such as "virt-who -d -o",
will start virt-who by service when no cli configured.
:param wait: wait time after run virt-who
:return: analyzer data dict
"""
for i in range(4):
rhsm_output = self.thread_start(cli, wait)
if self.msg_search(rhsm_output, "status=429"):
wait_time = 60 * (i + 3)
logger.warning(
f"429 code found, re-register virt-who host and try again "
f"after {wait_time} seconds...")
# self.re_register() need to be defined here later
time.sleep(wait_time)
elif self.msg_search(
rhsm_output, "RemoteServerException: Server error "
"attempting a GET.*returned status 500"):
logger.warning(
"RemoteServerException return 500 code, restart "
"virt-who again after 60s")
time.sleep(60)
else:
result_data = self.analyzer(rhsm_output)
return result_data
raise FailException("Failed to run virt-who service.")
def hostname_get(ssh):
"""
Get the hostname.
:param ssh: ssh access of host
"""
ret, output = ssh.runcmd('hostname')
if ret == 0 and output:
return output.strip()
raise FailException('Failed to get hostname.')
def selinux_disable(ssh):
"""
Disable selinux for one host.
:param ssh: ssh access of host
"""
ret, _ = ssh.runcmd("setenforce 0;"
"sed -i -e 's/SELINUX=.*/SELINUX=disabled/g' "
"/etc/selinux/config")
if ret != 0:
raise FailException('Failed to disable selinux')
def etc_hosts_set(ssh, value):
"""
Set the /etc/hosts file.
:param ssh: ssh access of host
:param value: should be the format of '{ip} {hostname}'
"""
ret, _ = ssh.runcmd(f"sed -i '/localhost/!d' /etc/hosts;"
f"echo '{value}' >> /etc/hosts")
if ret != 0:
raise FailException('Failed to set /etc/hosts')
def ipaddr_get(ssh):
"""
Get the host ip address.
:param ssh: ssh access of host
"""
ret, output = ssh.runcmd("ip route get 8.8.8.8 |"
"awk '/src/ { print $7 }'")
if ret == 0 and output:
return output.strip()
raise FailException(f'Failed to get ip address.')
def unregister(self):
"""
Unregister and clean host by subscription-manager.
"""
ret, _ = self.ssh.runcmd('subscription-manager unregister;'
'subscription-manager clean')
if ret == 0:
logger.info(f'Succeeded to unregister host')
else:
raise FailException(f'Failed to unregister {self.host}')
def satellite_cert_install(self):
"""
Install certificate when registering to satellite.
"""
cmd = f'rpm -ihv http://{self.server}' \
f'/pub/katello-ca-consumer-latest.noarch.rpm'
ret, _ = self.ssh.runcmd(cmd)
if ret != 0:
raise FailException(
f'Failed to install satellite certification for {self.host}')
def job_id_get():
"""
Get the job id of polarion upload
"""
ret, output = subprocess.getstatusoutput(
"cat %s | awk '{print $4}' | awk '$1=$1'" % args.log_file)
if ret == 0:
logger.info(f'Succeeded to get the polarion job id: {output}')
return output
raise FailException('Fail to get polarion job id')
def libvirt_access_no_password(ssh):
"""
Configure virt-who host accessing remote libvirt host by ssh
without password.
:param ssh: ssh access of virt-who host
"""
ssh_libvirt = SSHConnect(host=config.libvirt.server,
user=config.libvirt.username,
pwd=config.libvirt.password)
_, _ = ssh.runcmd('echo -e "\n" | ' 'ssh-keygen -N "" &> /dev/null')
ret, output = ssh.runcmd('cat ~/.ssh/id_rsa.pub')
if ret != 0 or output is None:
raise FailException("Failed to create ssh key")
_, _ = ssh_libvirt.runcmd(f"mkdir ~/.ssh/;"
f"echo '{output}' >> ~/.ssh/authorized_keys")
ret, _ = ssh.runcmd(f'ssh-keyscan -p 22 {config.libvirt.server} >> '
f'~/.ssh/known_hosts')
if ret != 0:
raise FailException(
'Failed to configure access libvirt without passwd')
def virtwho_install_by_url(ssh, url):
"""
Install virt-who package by a designated url.
:param ssh: ssh access of virt-who host
:param url: virt-who package url, whick can be local installed.
"""
if not base.url_validation(url):
raise FailException(f'package {url} is not available')
ssh.runcmd('rm -rf /var/cache/yum/;'
'yum clean all;'
'yum remove -y virt-who')
ssh.runcmd(f'yum localinstall -y {url}')
def rhel_version(ssh):
"""
Check the rhel version.
:param ssh: ssh access of host
:return: one number, such as 7, 8, 9
"""
ret, output = ssh.runcmd('cat /etc/redhat-release')
if ret == 0 and output:
m = re.search(r'(?<=release )\d', output)
rhel_ver = m.group(0)
return str(rhel_ver)
raise FailException('Failed to check rhel version.')
def firewall_stop(ssh):
"""
Stop firewall for one host.
:param ssh: ssh access of host
"""
cmd = ('systemctl stop firewalld.service;'
'systemctl disable firewalld.service')
if rhel_version(ssh) == '6':
cmd = 'service iptables stop; chkconfig iptables off'
ret, _ = ssh.runcmd(cmd)
if ret != 0:
raise FailException('Failed to stop firewall')
def satellite_installer(ssh, admin_password):
"""
Run command to deploy satellite by satellite-installer.
:param ssh: ssh access to satellite host.
:param admin_password: password for admin account.
"""
ret, output = ssh.runcmd(
f'satellite-installer --scenario satellite '
f'--disable-system-checks '
f'--foreman-initial-admin-password={admin_password}')
if ret != 0:
raise FailException('Failed to run satellite-installer')
def refresh(self):
"""
Refresh subscription by command 'subscription-manager refresh'.
"""
for i in range(3):
ret, output = self.ssh.runcmd('subscription-manager refresh')
if ret == 0:
logger.info(f'Succeeded to refresh subscription')
return True
logger.warning('Try again to refresh subscription after 180s...')
time.sleep(180)
raise FailException(f'Failed to refresh subscription for {self.host}')
def facts_remove(self):
"""
Remove subscription facts.
"""
ret, output = self.ssh.runcmd('rm -f /etc/rhsm/facts/custom.facts;'
'subscription-manager facts --update')
time.sleep(60)
if ret == 0 and 'Successfully updated' in output:
logger.info(f'Succeeded to remove custom.facts for {self.host}')
else:
raise FailException(
f'Failed to remove custom.facts for {self.host}')
def url_file_download(ssh, local_file, url):
"""
Test the url existence.
:param ssh: ssh access of host
:param local_file: local file path and name
:param url: url link of remote file
"""
_, _ = ssh.runcmd(f'rm -f {local_file};'
f'curl -L {url} -o {local_file};'
f'sync')
ret, output = ssh.runcmd(f'cat {local_file}')
if ret != 0 or 'Not Found' in output:
raise FailException(f'Failed to download {url}')
def unattach(self, pool=None):
"""
Remove subscription by Pool ID or remove all.
:param pool: Pool ID, remove all when pool=None.
"""
cmd = 'subscription-manager remove --all'
if pool:
cmd = f'subscription-manager remove --pool={pool}'
ret, output = self.ssh.runcmd(cmd)
if ret == 0:
logger.info(f'Succeeded to remove subscription for {self.host}')
else:
raise FailException(f'Failed to remove subscription for {self.host}')
def satellite_settings(ssh, name, value):
"""
Update the settings by hammer command.
:param ssh: ssh access to satellite host.
:param name: such as unregister_delete_host.
:param value: the value.
:return: True or raise Fail.
"""
ret, output = ssh.runcmd(f'hammer settings set '
f'--name={name} '
f'--value={value}')
if ret == 0 and f'Setting [{name}] updated to' in output:
return True
raise FailException(f'Failed to set {name}:{value} for satellite')
def uuid(self, host_name):
"""
Get the consumer uuid by host name
:param host_name: host name
:return: consumer uuid or None
"""
consumer = self.consumers(host_name)
if consumer:
uuid = consumer['uuid']
logger.info(f'Succeeded to get stage consumer uuid: '
f'{host_name}:{uuid}')
return uuid
raise FailException(
f'Failed to get stage consumer uuid for {host_name}')
def settings(self, name, value):
"""
Update the settings.
:param name: such as unregister_delete_host.
:param value: the value.
:return: True or raise Fail.
"""
ret, output = self.ssh.runcmd(f'hammer settings set '
f'--name={name} '
f'--value={value}')
if ret == 0 and f'Setting [{name}] updated to' in output:
logger.info(f'Succeeded to set {name}:{value} for satellite')
return True
raise FailException(f'Failed to set {name}:{value} for satellite')
def info(self, host_name):
"""
Get the consumer host information by host name, including the
detail facts info.
:param host_name: host name
:return: output to a dic
"""
uuid = self.uuid(host_name)
status, info = request_get(url=f'{self.api}/consumers/{uuid}',
auth=self.auth)
if status == 200:
logger.info(f'Succeeded to get consumer info for {host_name}')
return info
raise FailException(f'Failed to get consumer info for {host_name}')
def satellite_pkg_install(ssh):
"""
Run command to install satellite package.
:param ssh: ssh access to satellite host.
"""
# clean yum history and rebuilddb
ssh.runcmd('rm -f /var/lib/rpm/__db*;'
'rpm --rebuilddb;'
'rm -rf /var/lib/yum/history/*.sqlite;'
'rm -rf /var/cache/yum/*;'
'yum clean all;'
'rm -rf /etc/yum.repos.d/beaker*')
ret, output = ssh.runcmd('yum install -y satellite')
if ret != 0:
raise FailException('Failed to install satellite package')