def wait_till_rhevm4_instance_status(instance_name, status, timeout=5):
"""Waits untill given RHEVM4 VM status reached.
The following environment variables affect this command:
RHEV_USER
The username of a rhevm project to login.
RHEV_PASSWD
The password of a rhevm project to login.
RHEV_URL
An url to API of rhevm project.
:param instance_name: A string. RHEVM Instance name to create.
:param template_name: A string. RHEVM image name from which instance
to be created.
:param int timeout: The polling timeout in minutes to create rhevm
instance.
"""
with get_rhevm4_client().build() as rhevm_client:
timeup = time.time() + int(timeout) * 60
vservice = rhevm_client.system_service().vms_service()
while True:
if time.time() > timeup:
logger.warning(
'Timeout in turning VM instance {0}.'.format(status))
sys.exit(1)
vm = vservice.list(search='name={}'.format(instance_name))[0]
vm_status = vm.status.name
logger.info('Current Status: {0}'.format(vm_status))
if vm_status.lower() == status.lower():
return True
time.sleep(5)
def wait_till_rhevm_instance_status(instance_name, status, timeout=5):
"""Waits untill given VM status reached.
The following environment variables affect this command:
RHEV_USER
The username of a rhevm project to login.
RHEV_PASSWD
The password of a rhevm project to login.
RHEV_URL
An url to API of rhevm project.
:param instance_name: A string. RHEVM Instance name to create.
:param template_name: A string. RHEVM image name from which instance
to be created.
:param int timeout: The polling timeout in minutes to create rhevm
instance.
"""
rhevm_client = get_rhevm_client()
timeup = time.time() + int(timeout) * 60
while True:
if time.time() > timeup:
logger.warning(
'Timeout in turning VM instance {0}.'.format(status))
sys.exit(1)
vm_status = rhevm_client.vms.get(
name=instance_name).get_status().get_state()
logger.info('Current Status: {0}'.format(vm_status))
if vm_status == status:
return True
time.sleep(5)
rhevm_client.disconnect()
def satellite6_capsule_setup(sat_host, os_version, upgradable_capsule=True):
"""Setup all per-requisites for user provided capsule or auto created
capsule on rhevm for capsule upgrade.
:param string sat_host: Satellite hostname to which the capsule registered
:param string os_version: The OS version onto which the capsule installed
e.g: rhel6, rhel7
:param bool upgradable_capsule: Whether to setup capsule to be able to
upgrade in future
"""
# For User Defined Capsule
if os.environ.get('CAPSULE_HOSTNAMES'):
cap_hosts = os.environ.get('CAPSULE_HOSTNAMES')
if not os.environ.get('CAPSULE_AK'):
logger.warning('CAPSULE_AK environment variable is not defined !')
sys.exit(1)
# Else run upgrade on rhevm capsule
else:
# Get image name and Hostname from Jenkins environment
missing_vars = [
var
for var in ('RHEV_CAP_IMAGE', 'RHEV_CAP_HOST', 'RHEV_CAPSULE_AK')
if var not in os.environ
]
# Check if image name and Hostname in jenkins are set
if missing_vars:
logger.warning('The following environment variable(s) must be '
'set: {0}.'.format(', '.join(missing_vars)))
sys.exit(1)
cap_image = os.environ.get('RHEV_CAP_IMAGE')
cap_hosts = os.environ.get('RHEV_CAP_HOST')
cap_instance = 'upgrade_capsule_auto_{0}'.format(os_version)
execute(delete_rhevm_instance, cap_instance)
logger.info('Turning on Capsule Instance ....')
execute(create_rhevm_instance, cap_instance, cap_image)
non_responsive_host = []
env['capsule_hosts'] = cap_hosts
if ',' in cap_hosts:
cap_hosts = [cap.strip() for cap in cap_hosts.split(',')]
else:
cap_hosts = [cap_hosts]
for cap_host in cap_hosts:
if not host_pings(cap_host):
non_responsive_host.append(cap_host)
else:
execute(host_ssh_availability_check, cap_host)
execute(lambda: run('katello-service restart'), host=cap_host)
if non_responsive_host:
logger.warning(
str(non_responsive_host) + ' these are '
'non-responsive hosts')
sys.exit(1)
copy_ssh_key(sat_host, cap_hosts)
# Dont run capsule upgrade requirements for n-1 capsule
if upgradable_capsule:
execute(sync_capsule_repos_to_upgrade, cap_hosts, host=sat_host)
for cap_host in cap_hosts:
logger.info('Capsule {} is ready for Upgrade'.format(cap_host))
return cap_hosts
def satellite_capsule_setup(satellite_host,
capsule_hosts,
os_version,
upgradable_capsule=True):
"""
Setup all pre-requisites for user provided capsule
:param satellite_host: Satellite hostname to which the capsule registered
:param capsule_hosts: List of capsule which mapped with satellite host
:param os_version: The OS version onto which the capsule installed e.g: rhel6, rhel7
:param upgradable_capsule:Whether to setup capsule to be able to upgrade in future
:return: capsule_hosts
"""
if os_version == 'rhel6':
baseurl = settings.repos.rhel6_os
elif os_version == 'rhel7':
baseurl = settings.repos.rhel7_os
else:
logger.highlight('No OS Specified. Aborting...')
sys.exit(1)
non_responsive_host = []
for cap_host in capsule_hosts:
if not host_pings(cap_host):
non_responsive_host.append(cap_host)
else:
execute(host_ssh_availability_check, cap_host)
# Update the template once 1829115 gets fixed.
execute(workaround_1829115, host=cap_host)
if not bz_bug_is_open(1829115):
logger.warn(
"Please update the capsule template for fixed capsule version")
execute(foreman_service_restart, host=cap_host)
if non_responsive_host:
logger.highlight(
str(non_responsive_host) + ' these are non-responsive hosts. '
'Aborting...')
sys.exit(1)
copy_ssh_key(satellite_host, capsule_hosts)
if upgradable_capsule:
if settings.upgrade.distribution == "cdn":
settings.repos.capsule_repo = None
settings.repos.sattools_repo[settings.upgrade.os] = None
settings.repos.satmaintenance_repo = None
execute(update_capsules_to_satellite,
capsule_hosts,
host=satellite_host)
if settings.upgrade.upgrade_with_http_proxy:
http_proxy_config(capsule_hosts)
execute(sync_capsule_repos_to_satellite,
capsule_hosts,
host=satellite_host)
for cap_host in capsule_hosts:
settings.upgrade.capsule_hostname = cap_host
execute(add_baseOS_repo, baseurl, host=cap_host)
execute(yum_repos_cleanup, host=cap_host)
logger.info(f'Capsule {cap_host} is ready for Upgrade')
return capsule_hosts
def validate_and_create_product_templates(product):
"""Task to do a sanity check on the satellite and capsule and then
create their templates after z-stream upgrade
Environment variables required to run upgrade on RHEVM Setup and will be
fetched from Jenkins:
----------------------------------------------------------------------
RHEV_SAT_HOST
The rhevm satellite hostname to run upgrade on
RHEV_CAP_HOST
The rhevm capsule hostname to run upgrade on
RHEV_STORAGE
The storage domain on the rhevm used to create templates
RHEV_CLUSTER
Cluster name in RHEV where instance is created
RHEV_SAT_IMAGE
The satellite Image from which satellite instance will be created
RHEV_CAP_IMAGE
The capsule Image from which capsule instance will be created
RHEV_SAT_INSTANCE
The satellite instance name in rhevm of which template is to be
created, generally the upgraded box
RHEV_CAP_INSTANCE
The capsule instance name in rhevm of which template is to be
created, generally the upgraded box
"""
# Get the instances name, specified in the jenkins job
if product not in ['satellite', 'n-1']:
os_version = os.environ.get('OS_VERSION')
sat_instance = 'upgrade_satellite_auto_rhel{0}'.format(os_version)
logger.info('Satellite Instance name {0}'.format(sat_instance))
cap_instance = 'upgrade_capsule_auto_rhel{0}'.format(os_version)
logger.info('Capsule Instance name {0}'.format(cap_instance))
cluster = os.environ.get('RHEV_CLUSTER')
storage = os.environ.get('RHEV_STORAGE')
sat_host = os.environ.get('RHEV_SAT_HOST')
new_sat_template = os.environ.get('RHEV_SAT_IMAGE') + "_new"
cap_host = os.environ.get('RHEV_CAP_HOST')
new_cap_template = os.environ.get('RHEV_CAP_IMAGE') + "_new"
if check_necessary_env_variables_for_upgrade('capsule'):
execute(check_ntpd, host=sat_host)
execute(katello_restart, host=sat_host)
execute(capsule_sync, cap_host, host=sat_host)
execute(check_ntpd, host=cap_host)
execute(katello_restart, host=cap_host)
thread.start_new_thread(
create_rhevm_template,
(sat_instance, cluster, new_sat_template, storage))
thread.start_new_thread(
create_rhevm_template,
(cap_instance, cluster, new_cap_template, storage))
wait_till_rhevm_instance_status(sat_instance,
'Image Locked',
timeout=30)
wait_till_rhevm_instance_status(sat_instance, 'down', timeout=240)
def satellite6_capsule_zstream_upgrade(cap_host):
"""Upgrades Capsule to its latest zStream version
:param string cap_host: Capsule hostname onto which the capsule upgrade
will run
Note: For zstream upgrade both 'To' and 'From' version should be same
FROM_VERSION
Current satellite version which will be upgraded to latest version
TO_VERSION
Next satellite version to which satellite will be upgraded
"""
logger.highlight('\n========== CAPSULE UPGRADE =================\n')
from_version = os.environ.get('FROM_VERSION')
to_version = os.environ.get('TO_VERSION')
if not from_version == to_version:
logger.warning('zStream Upgrade on Capsule cannot be performed as '
'FROM and TO versions are not same!')
sys.exit(1)
major_ver = distro_info()[1]
if os.environ.get('CAPSULE_URL'):
disable_old_repos('rhel-{0}-server-satellite-capsule-{1}-rpms'.format(
major_ver, from_version))
# Check what repos are set
run('yum repolist')
if from_version == '6.0':
# Stop katello services, except mongod
run('for i in qpidd pulp_workers pulp_celerybeat '
'pulp_resource_manager httpd; do service $i stop; done')
else:
# Stop katello services
run('katello-service stop')
run('yum clean all', warn_only=True)
logger.info('Updating system and capsule packages ... ')
preyum_time = datetime.now().replace(microsecond=0)
update_packages(quiet=False)
postyum_time = datetime.now().replace(microsecond=0)
logger.highlight('Time taken for capsule packages update - {}'.format(
str(postyum_time - preyum_time)))
setup_capsule_firewall()
preup_time = datetime.now().replace(microsecond=0)
if to_version == '6.0':
run('katello-installer --upgrade')
elif to_version == '6.1':
run('capsule-installer --upgrade')
else:
run('satellite-installer --scenario capsule --upgrade ')
postup_time = datetime.now().replace(microsecond=0)
logger.highlight('Time taken for Capsule Upgrade - {}'.format(
str(postup_time - preup_time)))
# Rebooting the capsule for kernel update if any
reboot(160)
host_ssh_availability_check(cap_host)
# Check if Capsule upgrade is success
run('katello-service status', warn_only=True)
def satellite_capsule_zstream_upgrade(cap_host):
"""Upgrades Capsule to its latest zStream version
:param string cap_host: Capsule hostname onto which the capsule upgrade
will run
Note: For zstream upgrade both 'To' and 'From' version should be same
FROM_VERSION
Current satellite version which will be upgraded to latest version
TO_VERSION
Next satellite version to which satellite will be upgraded
"""
logger.highlight('\n========== CAPSULE UPGRADE =================\n')
from_version = settings.upgrade.from_version
to_version = settings.upgrade.to_version
if not from_version == to_version:
logger.highlight(
'zStream Upgrade on Capsule cannot be performed as FROM and TO '
'versions are not same. Aborting...')
sys.exit(1)
major_ver = settings.upgrade.os[-1]
ak_name = settings.upgrade.capsule_ak[settings.upgrade.os]
run(f'subscription-manager register --org="Default_Organization" '
f'--activationkey={ak_name} --force')
logger.info(
f"Activation key {ak_name} registered capsule's all available repository"
)
run("subscription-manager repos --list")
capsule_repos = [
RHEL_CONTENTS["tools"]["label"], RHEL_CONTENTS["capsule"]["label"],
RHEL_CONTENTS["maintenance"]["label"]
]
with fabric_settings(warn_only=True):
if settings.upgrade.distribution == "cdn":
enable_disable_repo(enable_repos_name=capsule_repos)
else:
enable_disable_repo(disable_repos_name=capsule_repos)
ansible_repos = [
f"rhel-{major_ver}-server-ansible-"
f"{settings.upgrade.ansible_repo_version}-rpms"
]
enable_disable_repo(enable_repos_name=ansible_repos)
# Check what repos are set
# setup_foreman_maintain_repo()
if settings.upgrade.foreman_maintain_capsule_upgrade:
upgrade_using_foreman_maintain(sat_host=False)
else:
nonfm_upgrade(satellite_upgrade=False)
# Rebooting the capsule for kernel update if any
if settings.upgrade.satellite_capsule_setup_reboot:
reboot(160)
host_ssh_availability_check(cap_host)
# Check if Capsule upgrade is success
upgrade_validation()
def wait_untill_capsule_sync(capsule):
"""The polling function that waits for capsule sync task to finish
:param capsule: A capsule hostname
"""
cap = entities.Capsule().search(
query={'search': 'name={}'.format(capsule)})[0]
active_tasks = cap.content_get_sync()['active_sync_tasks']
if len(active_tasks) >= 1:
logger.info('Wait for background capsule sync to finish on '
'capsule: {}'.format(cap.name))
for task in active_tasks:
entities.ForemanTask(id=task['id']).poll(timeout=2700)
def docker_clients_upgrade(old_repo, clients, agent):
"""Helper function to run upgrade on docker containers as clients
:param string old_repo: The old tools repo to disable before updating
katello-agent package
:param dict clients: The dictionary containing client_name as key and
container_id as value
:param string agent: puppet-agent / katello-agent
"""
for hostname, container in tuple(clients.items()):
logger.info(f'Upgrading client {hostname} on docker container: {container}')
docker_execute_command(container, f'subscription-manager repos --disable {old_repo}')
docker_execute_command(container, f'yum update -y {agent}', True)
def reboot(halt_time=300):
"""Reboots the host.
Also halts the execution until reboots according to given time.
:param int halt_time: Halt execution in seconds.
"""
halt_time = halt_time
logger.info('Rebooting the host, please wait .... ')
try:
run('reboot', warn_only=True)
except Exception as e:
logger.info(e)
time.sleep(halt_time)
def capsule_sync(cap_host):
"""Run Capsule Sync as a part of job
:param list cap_host: List of capsules to perform sync
"""
set_hammer_config()
if os.environ.get('TO_VERSION') in ['6.2', '6.3', '6.4']:
logger.info('Refreshing features for capsule host {0}'.
format(cap_host))
print hammer('capsule refresh-features --name "{0}"'.
format(cap_host))
logger.info('Running Capsule sync for capsule host {0}'.
format(cap_host))
print hammer('capsule content synchronize --name {0}'.format(cap_host))
def check_capsule(capsule_name):
"""Running capsule sync on external capsule"""
set_hammer_config()
if os.environ.get('TO_VERSION') in ['6.2', '6.3']:
check = hammer(
'capsule refresh-features --name "{0}"'.format(capsule_name))
print check[u'message']
if check.return_code == 0:
logger.info('Running Capsule sync')
hammer('capsule content synchronize --name "{0}"'.format(
capsule_name))
else:
logger.info('Running Capsule sync')
hammer('capsule content synchronize --name "{0}"'.format(capsule_name))
def docker_client_missing_package_installation(clients, agent):
"""
Use to install the missing packages of puppet agent and katello-agent
:param dict clients: The dictionary containing client_name as key and
container_id as value
:param string agent: puppet-agent / katello-agent
"""
for hostname, container in tuple(clients.items()):
logger.info(f'Installing client {hostname} on docker container: {container}')
try:
for _ in range(1, 15):
yum_status = docker_execute_command(container, 'pgrep yum', True)
if yum_status == '':
break
logger.info("yum command is running wait fot 60 seconds...")
time.sleep(60)
command_output = docker_execute_command(container, f'rpm -q {agent}', True)
if re.search(f'package {agent} is not installed', command_output):
logger.warn(f"base version of {agent} package missed(because of timeout) on "
f"{container} so installing it separately")
docker_execute_command(container, f'yum install -y {agent}', True)
command_output = docker_execute_command(container, f'rpm -q {agent}', True)
if re.search(f'package {agent} is not installed', command_output):
logger.warn(f"failed to install package {agent} on {container}")
else:
logger.info(f"base version of {agent} package {command_output} installed "
f"successfully on {container}")
else:
logger.info(f"{agent} package {command_output} is available before "
f"upgrade on {container}")
except Exception as ex:
logger.warn(ex)
def setup_products_for_upgrade(product, os_version):
"""Sets up product(s) to perform upgrade on
:param string product: The product name to setup before upgrade
:param string os_version: The os version on which product is installed
e.g: rhel6, rhel7
"""
if check_necessary_env_variables_for_upgrade(product):
sat_host = cap_hosts = clients6 = clients7 = None
logger.info('Setting up Satellite ....')
sat_host = satellite6_setup(os_version)
if product == 'capsule' or product == 'n-1' or product == 'longrun':
logger.info('Setting up Capsule ....')
cap_hosts = satellite6_capsule_setup(
sat_host, os_version, False if product == 'n-1' else True)
if product == 'client' or product == 'longrun':
logger.info('Setting up Clients ....')
clients6, clients7 = satellite6_client_setup()
setups_dict = {
'sat_host': sat_host,
'capsule_hosts': cap_hosts,
'clients6': clients6,
'clients7': clients7
}
create_setup_dict(setups_dict)
if os.environ.get('RUN_EXISTANCE_TESTS', 'false').lower() == 'true':
logger.info('Setting up preupgrade datastore for existance tests')
set_datastore('preupgrade')
return sat_host, cap_hosts, clients6, clients7
def capsule_sync(cap_host):
"""Run Capsule Sync as a part of job
:param list cap_host: List of capsules to perform sync
"""
capsule = entities.SmartProxy().search(
query={'search': 'name={}'.format(cap_host)})[0]
if float(os.environ.get('TO_VERSION')) >= 6.2:
logger.info(
'Refreshing features for capsule host {0}'.format(cap_host))
capsule.refresh()
logger.info('Running Capsule sync for capsule host {0}'.format(cap_host))
capsule = entities.Capsule().search(
query={'search': 'name={}'.format(cap_host)})[0]
capsule.content_sync()
def satellite_setup(satellite_host):
"""
The purpose of this method to make the satellite ready for upgrade.
:param satellite_host:
:return: satellite_host
"""
execute(host_ssh_availability_check, satellite_host)
execute(yum_repos_cleanup, host=satellite_host)
execute(install_prerequisites, host=satellite_host)
execute(subscribe, host=satellite_host)
execute(foreman_service_restart, host=satellite_host)
env['satellite_host'] = satellite_host
settings.upgrade.satellite_hostname = satellite_host
logger.info(f'Satellite {satellite_host} is ready for Upgrade!')
return satellite_host
def docker_clients_upgrade(old_repo, clients):
"""Helper function to run upgrade on docker containers as clients
:param string old_repo: The old tools repo to disable before updating
katello-agent package
:param dict clients: The dictionary containing client_name as key and
container_id as value
"""
for hostname, container in clients.items():
logger.info('Upgrading client {0} on docker container: {1}'.format(
hostname, container))
docker_execute_command(
container,
'subscription-manager repos --disable {}'.format(old_repo))
docker_execute_command(container, 'yum update -y katello-agent', True)
def docker_cleanup_containers():
logger.info('Cleaning UP of Docker containers BEGINS')
logger.info('Stopping all the running docker containers')
run('docker ps -a | grep \'days ago\' | awk \'{print $1}\' | xargs '
'--no-run-if-empty docker stop')
logger.info('Removing all the docker containers')
run('docker ps -a | grep \'days ago\' | awk \'{print $1}\' | xargs '
'--no-run-if-empty docker rm ')
logger.info('Cleaning UP of Docker containers ENDS')
def docker_clients_upgrade(old_repo, clients, puppet=False):
"""Helper function to run upgrade on docker containers as clients
:param string old_repo: The old tools repo to disable before updating
katello-agent package
:param dict clients: The dictionary containing client_name as key and
container_id as value
:param bool puppet: clients are puppet clients or not, default no
"""
agent = 'puppet' if puppet else 'katello'
for hostname, container in clients.items():
logger.info('Upgrading client {0} on docker container: {1}'.format(
hostname, container))
docker_execute_command(
container,
'subscription-manager repos --disable {}'.format(old_repo))
docker_execute_command(container,
'yum update -y {}-agent'.format(agent), True)
def nonfm_upgrade(satellite_upgrade=True, cap_host=None, sat_host=None):
"""
The purpose of this module to perform the upgrade task without foreman-maintain.
In this function we setup the repository, stop the katello services,
cleanup, and execute satellite upgrade task"
:param bool satellite_upgrade: If satellite_upgrade is True then upgrade
type satellite otherwise capsule
:param bool zstream: Capsule zStream upgrade
:param str cap_host: hostname of capsule it used to generate certificate for
capsules major version upgrade.
:param str sat_host: hostname of satellite used to generate certificate for
capsules major version upgrade.
:
"""
# Check what repos are set
upgrade_type = "satellite" if satellite_upgrade else "capsule"
run('yum repolist')
# Stop katello services, except mongod
run('katello-service stop')
run('yum clean all', warn_only=True)
# Updating the packages again after setting sat6 repo
logger.info('Updating system and {} packages... '.format(upgrade_type))
preyum_time = datetime.now().replace(microsecond=0)
update_packages(quiet=False)
postyum_time = datetime.now().replace(microsecond=0)
logger.highlight('Time taken for system and {} packages update'
' - {}'.format(upgrade_type,
str(postyum_time - preyum_time)))
# non zStream capsule upgrade
if sat_host and cap_host:
execute(generate_capsule_certs, cap_host, True, host=sat_host)
execute(lambda: run("scp -o 'StrictHostKeyChecking no' {0}-certs.tar "
"root@{0}:/home/".format(cap_host)),
host=sat_host)
setup_capsule_firewall()
preup_time = datetime.now().replace(microsecond=0)
upgrade_task(upgrade_type, cap_host)
else:
preup_time = datetime.now().replace(microsecond=0)
upgrade_task(upgrade_type)
postup_time = datetime.now().replace(microsecond=0)
logger.highlight('Time taken for Satellite Upgrade - {}'.format(
str(postup_time - preup_time)))
def satellite_setup(satellite_host):
"""
The purpose of this method to make the satellite ready for upgrade.
:param satellite_host:
:return: satellite_host
"""
execute(host_ssh_availability_check, satellite_host)
execute(yum_repos_cleanup, host=satellite_host)
execute(install_prerequisites, host=satellite_host)
execute(subscribe, host=satellite_host)
execute(foreman_service_restart, host=satellite_host)
maintenance_repo_update()
env['satellite_host'] = satellite_host
settings.upgrade.satellite_hostname = satellite_host
execute(hammer_config, host=satellite_host)
# remove the workaround after 2031154 fixes
if settings.upgrade.from_version == '6.10':
execute(workaround_2031154, host=satellite_host)
logger.info(f'Satellite {satellite_host} is ready for Upgrade!')
return satellite_host
def satellite6_setup(os_version):
"""Sets up required things on upgrade running machine and on Satellite to
perform satellite upgrade later
:param string os_version: The OS version onto which the satellite installed
e.g: rhel6, rhel7
"""
# If Personal Satellite Hostname provided
if os.environ.get('SATELLITE_HOSTNAME'):
sat_host = os.environ.get('SATELLITE_HOSTNAME')
# Else run upgrade on rhevm satellite
else:
# Get image name and Hostname from Jenkins environment
missing_vars = [
var for var in ('RHEV_SAT_IMAGE', 'RHEV_SAT_HOST')
if var not in os.environ
]
# Check if image name and Hostname in jenkins are set
if missing_vars:
logger.warning('The following environment variable(s) must be set '
'in jenkins environment: {0}.'.format(
', '.join(missing_vars)))
sys.exit(1)
sat_image = os.environ.get('RHEV_SAT_IMAGE')
sat_host = os.environ.get('RHEV_SAT_HOST')
sat_instance = 'upgrade_satellite_auto_{0}'.format(os_version)
execute(delete_rhevm4_instance, sat_instance)
execute(create_rhevm4_instance, sat_instance, sat_image)
if not host_pings(sat_host):
sys.exit(1)
execute(host_ssh_availability_check, sat_host)
# start's/enables/install's ntp
# Check that hostname and localhost resolve correctly
execute(install_prerequisites, host=sat_host)
# Subscribe the instance to CDN
execute(subscribe, host=sat_host)
execute(foreman_service_restart, host=sat_host)
# Set satellite hostname in fabric environment
env['satellite_host'] = sat_host
logger.info('Satellite {} is ready for Upgrade!'.format(sat_host))
return sat_host
def create_rhevm_instance(instance_name,
template_name,
datacenter='Default',
quota='admin',
cluster='Default',
timeout=5):
"""Creates rhevm Instance from template.
The assigning template should have network and storage configuration saved
already.
ssh_key should be added to openstack project before running automation.
Else the automation will fail.
The following environment variables affect this command:
RHEV_USER
The username of a rhevm project to login.
RHEV_PASSWD
The password of a rhevm project to login.
RHEV_URL
An url to API of rhevm project.
:param instance_name: A string. RHEVM Instance name to create.
:param template_name: A string. RHEVM image name from which instance
to be created.
:param int timeout: The polling timeout in minutes to create rhevm
instance.
"""
rhevm_client = get_rhevm_client()
template = rhevm_client.templates.get(name=template_name)
datacenter = rhevm_client.datacenters.get(name=datacenter)
quota = datacenter.quotas.get(name=quota)
logger.info('Turning on instance {0} from template {1}. Please wait '
'till get up ...'.format(instance_name, template_name))
rhevm_client.vms.add(
params.VM(name=instance_name,
cluster=rhevm_client.clusters.get(name=cluster),
template=template,
quota=quota))
if wait_till_rhevm_instance_status(instance_name, 'down', timeout=timeout):
rhevm_client.vms.get(name=instance_name).start()
if wait_till_rhevm_instance_status(instance_name,
'up',
timeout=timeout):
logger.info('Instance {0} is now up !'.format(instance_name))
# We can fetch the Instance FQDN only if RHEV-agent is installed.
# Templates under SAT-QE datacenter includes RHEV-agents.
if rhevm_client.datacenters.get(name='SAT-QE'):
# get the hostname of instance
vm_fqdn = rhevm_client.vms.get(
name=instance_name).get_guest_info().get_fqdn()
logger.info('\t Instance FQDN : %s' % (vm_fqdn))
# We need value of vm_fqdn so that we can use it with CI
# For now, we are exporting it as a variable value
# and source it to use via shell script
file_path = "/tmp/rhev_instance.txt"
with open(file_path, 'w') as f1:
f1.write('export SAT_INSTANCE_FQDN={0}'.format(vm_fqdn))
rhevm_client.disconnect()
def setup_products_for_upgrade(product, os_version):
"""Sets up product(s) to perform upgrade on
:param string product: The product name to setup before upgrade
:param string os_version: The os version on which product is installed
e.g: rhel6, rhel7
"""
if check_necessary_env_variables_for_upgrade(product):
sat_host = cap_hosts = clients6 = clients7 = None
logger.info('Setting up Satellite ....')
sat_host = satellite6_setup(os_version)
if product == 'capsule' or product == 'n-1' or product == 'longrun':
logger.info('Setting up Capsule ....')
cap_hosts = satellite6_capsule_setup(
sat_host, os_version, False if product == 'n-1' else True)
if product == 'client' or product == 'longrun':
logger.info('Setting up Clients ....')
clients6, clients7, puppet_clients7 = satellite6_client_setup()
setups_dict = {
'sat_host': sat_host,
'capsule_hosts': cap_hosts,
'clients6': clients6,
'clients7': clients7,
'puppet_clients7': puppet_clients7
}
create_setup_dict(setups_dict)
return sat_host, cap_hosts, clients6, clients7, puppet_clients7
def get_hostname_from_ip(ip, timeout=3):
"""Retrives the hostname by logging into remote machine by IP.
Specially for the systems who doesnt support reverse DNS.
e.g usersys machines.
:param ip: A string. The IP address of the remote host.
:param int timeout: The polling timeout in minutes.
"""
timeup = time.time() + int(timeout) * 60
while True:
if time.time() > timeup:
logger.warning(
'The timeout for getting the Hostname from IP has reached!')
return False
try:
output = execute(lambda: run('hostname'), host=ip)
logger.info('Hostname determined as: {0}'.format(output[ip]))
break
except:
time.sleep(5)
return output[ip]
def post_upgrade_test_tasks(sat_host, cap_host=None):
"""Run set of tasks for post upgrade tests
:param string sat_host: Hostname to run the tasks on
:param list cap_host: Capsule hosts to run sync on
"""
# Execute tasks as post upgrade tests are dependent
certificate_url = os.environ.get('FAKE_MANIFEST_CERT_URL')
if certificate_url is not None:
execute(setup_fake_manifest_certificate,
certificate_url,
host=sat_host)
sat_version = os.environ.get('TO_VERSION')
execute(setup_alternate_capsule_ports, host=sat_host)
if float(sat_version) > 6.1:
# Update the Default Organization name, which was updated in 6.2
logger.info("Update the Default Organization name, which was updated "
"in 6.2")
org = entities.Organization().search(
query={'search': 'label=Default_Organization'})[0]
org.name = "Default Organization"
org.update(['name'])
# Update the Default Location name, which was updated in 6.2
logger.info("Update the Default Location name, which was updated in "
"6.2")
loc = entities.Location().search(
query={'search': 'name="Default Location"'})[0]
loc.name = "Default Location"
loc.update(['name'])
if bz_bug_is_open(1502505):
logger.info(
"Update the default_location_puppet_content value with "
"updated location name.Refer BZ:1502505")
puppet_location = entities.Setting().search(
query={'search': 'name=default_location_puppet_content'})[0]
puppet_location.value = 'Default Location'
puppet_location.update(['value'])
# Increase log level to DEBUG, to get better logs in foreman_debug
execute(lambda: run('sed -i -e \'/:level: / s/: .*/: '
'debug/\' /etc/foreman/settings.yaml'),
host=sat_host)
execute(lambda: run('katello-service restart'), host=sat_host)
# Execute task for template changes required for discovery feature
execute(setup_foreman_discovery, sat_version=sat_version, host=sat_host)
# Execute task for creating latest discovery iso required for unattended
# test
env.disable_known_hosts = True
execute(get_discovery_image, host=os.environ.get('LIBVIRT_HOSTNAME'))
# Commenting out until GH issue:#135
# Removing the original manifest from Default Organization (Org-id 1),
# to allow test-cases to utilize the same manifest.
# logger.info("Removing the Original Manifest from Default Organization")
# execute(hammer, 'subscription delete-manifest --organization-id 1',
# host=sat_host)
# Run Avahi Task on upgrade boxes for REX tests to run
execute(lambda: run('yum remove -y epel*'), host=sat_host)
execute(setup_avahi_discovery, host=sat_host)
def delete_openstack_instance(instance_name):
"""Deletes openstack Instance.
:param instance_name: A string. Openstack instance name to delete.
The following environment variables affect this command:
USERNAME
The username of an openstack project to login.
PASSWORD
The password of an openstack project to login.
AUTH_URL
The authentication url of the project.
PROJECT_NAME
Project NAME of an openstack project.
"""
openstack_client = get_openstack_client()
if openstack_client.delete_server(instance_name, timeout=300):
logger.info('Success! The instance {0} has been deleted from '
'Openstack.'.format(instance_name))
else:
logger.error('Instance {0} not found in Openstack project.'.format(
instance_name))
def dockerize(ak_name=None, distro=None, org_label=None):
"""Creates Docker Container's of specified distro and subscribes them to
given AK
:param string ak_name : Activation Key name, to be used to subscribe
the docker container's
:param string distro : The OS of the VM to be created.
Supported are 'rhel7' and 'rhel6'
:returns dict clients : A dictonary which contain's container name
and id.
Environment Variable:
DOCKER_VM
The Docker VM IP/Hostname on rhevm to create clients
RHEV_CLIENT_AK
The AK using which client will be registered to satellite
"""
ak_name = ak_name or os.environ.get(
'RHEV_CLIENT_AK_{}'.format(distro.upper()))
docker_vm = os.environ.get('DOCKER_VM')
# Check if the VM containing docker images is up, else turn on
with get_rhevm4_client().build() as rhevm_client:
instance_name = 'sat6-docker-upgrade'
template_name = 'sat6-docker-upgrade-template'
vm = rhevm_client.system_service().vms_service(
).list(search='name={}'.format(instance_name))
if not vm:
logger.info('Docker VM for generating Content Host is not created.'
'Creating it, please wait..')
create_rhevm4_instance(instance_name, template_name)
execute(manage_daemon, 'restart', 'docker', host=docker_vm)
elif vm[0].status.name.lower() == 'down':
logger.info('Docker VM for generating Content Host is not up. '
'Turning on, please wait ....')
rhevm_client.vms.get(name=instance_name).start()
wait_till_rhevm4_instance_status(instance_name, 'up', 5)
execute(manage_daemon, 'restart', 'docker', host=docker_vm)
time.sleep(5)
logger.info('Generating katello client on RHEL7 on Docker. '
'Please wait .....')
# Generate Clients on RHEL 7
time.sleep(30)
clients = execute(
generate_satellite_docker_clients_on_rhevm,
distro,
1,
ak_name,
org_label,
host=docker_vm,
)[docker_vm]
return clients
def post_upgrade_test_tasks(sat_host, cap_host=None):
"""Run set of tasks for post upgrade tests
:param string sat_host: Hostname to run the tasks on
:param list cap_host: Capsule hosts to run sync on
"""
# Execute tasks as post upgrade tests are dependent
certificate_url = os.environ.get('FAKE_MANIFEST_CERT_URL')
if certificate_url is not None:
execute(
setup_fake_manifest_certificate,
certificate_url,
host=sat_host
)
sat_version = os.environ.get('TO_VERSION')
execute(setup_alternate_capsule_ports, host=sat_host)
if sat_version not in ['6.0', '6.1']:
# Update the Default Organization name, which was updated in 6.2
logger.info("Update the Default Organization name, which was updated "
"in 6.2")
execute(hammer, 'organization update --name "Default_Organization" '
'--new-name "Default Organization" ',
host=sat_host)
# Update the Default Location name, which was updated in 6.2
logger.info("Update the Default Location name, which was updated in "
"6.2")
execute(hammer, 'location update --name "Default_Location" '
'--new-name "Default Location" ',
host=sat_host)
if bz_bug_is_open(1502505):
logger.info(
"Update the default_location_puppet_content value with "
"updated location name.Refer BZ:1502505")
execute(hammer, 'settings set --name '
'"default_location_puppet_content" --value '
'"Default Location"', host=sat_host)
# Increase log level to DEBUG, to get better logs in foreman_debug
execute(lambda: run('sed -i -e \'/:level: / s/: .*/: '
'debug/\' /etc/foreman/settings.yaml'), host=sat_host)
execute(lambda: run('katello-service restart'), host=sat_host)
# Execute capsule sync task , after the upgrade is completed
if cap_host:
execute(capsule_sync, cap_host, host=sat_host)
# Execute task for template changes required for discovery feature
execute(
setup_foreman_discovery,
sat_version=sat_version,
host=sat_host
)
def delete_rhevm4_instance(instance_name, timeout=5):
"""Deletes RHEVM4 Instance.
The following environment variables affect this command:
RHEV_USER
The username of a rhevm project to login.
RHEV_PASSWD
The password of a rhevm project to login.
RHEV_URL
An url to API of rhevm project.
:param instance_name: A string. RHEVM instance name to delete.
:param int timeout: The polling timeout in minutes to delete rhevm
instance.
"""
with get_rhevm4_client().build() as rhevm_client:
vservice = rhevm_client.system_service().vms_service()
vm = vservice.list(search='name={}'.format(instance_name))
if not vm:
logger.info('The instance {0} is not found '
'in RHEV to delete!'.format(instance_name))
else:
vm = vm[0]
logger.info(
'Deleting instance {0} from RHEVM.'.format(instance_name))
if vm.delete_protected:
logger.warning(
'The instance {0} is under delete protection and cannot '
'be deleted.'.format(instance_name))
sys.exit(1)
if vm.status.name == 'UP':
vservice.vm_service(vm.id).shutdown()
if wait_till_rhevm4_instance_status(instance_name, 'down'):
vservice.vm_service(vm.id).remove()
elif vm.status.name == 'DOWN':
vservice.vm_service(vm.id).remove()
timeup = time.time() + int(timeout) * 60
while True:
if time.time() > timeup:
logger.warning(
'The timeout for deleting RHEVM instance has reached!')
sys.exit(1)
if not vservice.list(search='name={}'.format(instance_name)):
logger.info(
'Instance {0} is now deleted from RHEVM!'.format(
instance_name))
break