def provision_appliances( self, count=1, preconfigured=False, version=None, stream=None, provider=None, lease_time=120, ram=None, cpu=None): # If we specify version, stream is ignored because we will get that specific version if version: stream = get_stream(version) # If we specify stream but not version, sprout will give us latest version of that stream elif stream: pass # If we dont specify either, we will get the same version as current appliance else: stream = get_stream(current_appliance.version) version = current_appliance.version.vstring request_id = self.call_method( 'request_appliances', preconfigured=preconfigured, version=version, group=stream, provider=provider, lease_time=lease_time, ram=ram, cpu=cpu, count=count ) wait_for( lambda: self.call_method('request_check', str(request_id))['finished'], num_sec=300, message='provision {} appliance(s) from sprout'.format(count)) data = self.call_method('request_check', str(request_id)) logger.debug(data) appliances = [] for appliance in data['appliances']: appliances.append(IPAppliance(hostname=appliance['ip_address'])) return appliances, request_id
def provision_appliances( self, count=1, preconfigured=False, version=None, stream=None, provider=None, provider_type=None, lease_time=120, ram=None, cpu=None, **kwargs): # provisioning may take more time than it is expected in some cases wait_time = kwargs.get('wait_time', 300) # If we specify version, stream is ignored because we will get that specific version if version: stream = get_stream(version) # If we specify stream but not version, sprout will give us latest version of that stream elif stream: pass # If we dont specify either, we will get the same version as current appliance else: stream = get_stream(current_appliance.version) version = current_appliance.version.vstring request_id = self.call_method( 'request_appliances', preconfigured=preconfigured, version=version, provider_type=provider_type, group=stream, provider=provider, lease_time=lease_time, ram=ram, cpu=cpu, count=count, **kwargs ) wait_for( lambda: self.call_method('request_check', str(request_id))['finished'], num_sec=wait_time, message='provision {} appliance(s) from sprout'.format(count)) data = self.call_method('request_check', str(request_id)) logger.debug(data) appliances = [] for appliance in data['appliances']: app_args = {'hostname': appliance['ip_address'], 'project': appliance['project'], 'container': appliance['container'], 'db_host': appliance['db_host']} appliances.append(IPAppliance(**app_args)) return appliances, request_id
def provision_appliances(self, count=1, preconfigured=False, version=None, stream=None, provider=None, provider_type=None, lease_time=60, ram=None, cpu=None, **kwargs): # provisioning may take more time than it is expected in some cases wait_time = kwargs.get('wait_time', 900) # If we specify version, stream is ignored because we will get that specific version if version: stream = get_stream(version) # If we specify stream but not version, sprout will give us latest version of that stream elif stream: pass # If we dont specify either, we will get the same version as current appliance else: stream = get_stream(current_appliance.version) version = current_appliance.version.vstring request_id = self.call_method('request_appliances', preconfigured=preconfigured, version=version, provider_type=provider_type, group=stream, provider=provider, lease_time=lease_time, ram=ram, cpu=cpu, count=count, **kwargs) wait_for(lambda: self.call_method('request_check', str(request_id))[ 'finished'], num_sec=wait_time, message='provision {} appliance(s) from sprout'.format(count)) data = self.call_method('request_check', str(request_id)) logger.debug(data) appliances = [] for appliance in data['appliances']: app_args = { 'hostname': appliance['ip_address'], 'project': appliance['project'], 'container': appliance['container'], 'db_host': appliance['db_host'] } appliances.append(IPAppliance(**app_args)) return appliances, request_id
def fqdn_appliance(appliance, preconfigured): sp = SproutClient.from_config() available_providers = set(sp.call_method('available_providers')) required_providers = set(cfme_data['fqdn_providers']) usable_providers = available_providers & required_providers version = appliance.version.vstring stream = get_stream(appliance.version) for provider in usable_providers: try: apps, pool_id = sp.provision_appliances( count=1, preconfigured=preconfigured, version=version, stream=stream, provider=provider ) break except Exception as e: logger.warning("Couldn't provision appliance with following error:") logger.warning("{}".format(e)) continue else: logger.error("Couldn't provision an appliance at all") raise SproutException('No provision available') yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def fqdn_appliance(appliance, preconfigured): sp = SproutClient.from_config() available_providers = set(sp.call_method('available_providers')) required_providers = set(cfme_data['fqdn_providers']) usable_providers = available_providers & required_providers version = appliance.version.vstring stream = get_stream(appliance.version) for provider in usable_providers: try: apps, pool_id = sp.provision_appliances( count=1, preconfigured=preconfigured, version=version, stream=stream, provider=provider) break except Exception as e: logger.warning( "Couldn't provision appliance with following error:") logger.warning("{}".format(e)) continue else: logger.error("Couldn't provision an appliance at all") raise SproutException('No provision available') yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def extdb_template(appliance): try: api = trackerbot.api() stream = get_stream(appliance.version.vstring) template_data = trackerbot.latest_template(api, stream) return api.template(template_data['latest_template'] + '-extdb').get() except BaseException: pytest.skip("trackerbot is unreachable")
def temp_appliance_preconfig_funcscope_upgrade(appliance): stream = ( int(''.join([i for i in get_stream(appliance.version) if i.isdigit()])) - 1) stream = "downstream-{}z".format(stream) with temp_appliances(preconfigured=True, stream=stream) as appliances: yield appliances[0]
def template_tags(appliance): try: api = trackerbot.api() stream = get_stream(appliance.version.vstring) template_data = trackerbot.latest_template(api, stream) template = api.template(template_data['latest_template']).get() # TODO: fix this in trackerbot by adding appropriate serialization to Template tags = json.loads(template['custom_data'].replace("'", '"').replace('u"', '"')) return tags['TAGS'] except BaseException: pytest.skip("trackerbot is unreachable")
def temp_appliance_preconfig_funcscope_upgrade(appliance, pytestconfig): stream_digits = ''.join( [i for i in get_stream(appliance.version) if i.isdigit()]) with sprout_appliances( appliance, config=pytestconfig, preconfigured=True, stream="downstream-{}z".format(int(stream_digits) - 1) # n-1 stream for upgrade ) as appliances: yield appliances[0]
def template_tags(appliance): try: api = trackerbot.api() stream = get_stream(appliance.version.vstring) template_data = trackerbot.latest_template(api, stream) template = api.template(template_data['latest_template']).get() # TODO: fix this in trackerbot by adding appropriate serialization to Template tags = json.loads(template['custom_data'].replace("'", '"').replace( 'u"', '"')) return tags['TAGS'] except BaseException: pytest.skip("trackerbot is unreachable")
def provision_appliances(self, count=1, preconfigured=False, version=None, stream=None, provider=None, provider_type=None, lease_time=120, ram=None, cpu=None): # If we specify version, stream is ignored because we will get that specific version if version: stream = get_stream(version) # If we specify stream but not version, sprout will give us latest version of that stream elif stream: pass # If we dont specify either, we will get the same version as current appliance else: stream = get_stream(current_appliance.version) version = current_appliance.version.vstring request_id = self.call_method('request_appliances', preconfigured=preconfigured, version=version, provider_type=provider_type, group=stream, provider=provider, lease_time=lease_time, ram=ram, cpu=cpu, count=count) wait_for(lambda: self.call_method('request_check', str(request_id))[ 'finished'], num_sec=300, message='provision {} appliance(s) from sprout'.format(count)) data = self.call_method('request_check', str(request_id)) logger.debug(data) appliances = [] for appliance in data['appliances']: appliances.append(IPAppliance(hostname=appliance['ip_address'])) return appliances, request_id
def fqdn_appliance(appliance, preconfigured, count): version = appliance.version.vstring stream = get_stream(appliance.version) try: sprout_client = SproutClient.from_config() apps, request_id = sprout_client.provision_appliances( provider_type='rhevm', count=count, version=version, preconfigured=preconfigured, stream=stream) yield apps finally: for app in apps: app.ssh_client.close() if request_id: sprout_client.destroy_pool(request_id)
def test_update_ha(ha_appliances_with_providers, appliance, update_strategy, request, old_version): """ Tests updating an appliance with providers using webui, also confirms that the provisioning continues to function correctly after the update has completed Polarion: assignee: jhenner caseimportance: high casecomponent: Appliance initialEstimate: 1/4h """ evm_log = '/var/www/miq/vmdb/log/evm.log' update_strategy(ha_appliances_with_providers[2]) wait_for(do_appliance_versions_match, func_args=(appliance, ha_appliances_with_providers[2]), num_sec=900, delay=20, handle_exception=True, message='Waiting for appliance to update') if BZ(1704835, forced_streams=get_stream( ha_appliances_with_providers[2].version)).blocks: with LogValidator( evm_log, matched_patterns=[r'Starting database failover monitor'], hostname=ha_appliances_with_providers[2].hostname).waiting( wait=30): ha_appliances_with_providers[2].evm_failover_monitor.restart() assert ha_appliances_with_providers[2].evm_failover_monitor.running with LogValidator( evm_log, matched_patterns=['Starting to execute failover'], hostname=ha_appliances_with_providers[2].hostname).waiting( wait=450): # Cause failover to occur result = ha_appliances_with_providers[0].ssh_client.run_command( 'systemctl stop $APPLIANCE_PG_SERVICE', timeout=15) assert result.success, "Failed to stop APPLIANCE_PG_SERVICE: {}".format( result.output) ha_appliances_with_providers[2].evmserverd.wait_for_running() ha_appliances_with_providers[2].wait_for_web_ui() # Verify that existing provider can detect new VMs virtual_crud = provider_app_crud(VMwareProvider, ha_appliances_with_providers[2]) vm = provision_vm(request, virtual_crud) assert vm.provider.mgmt.does_vm_exist(vm.name), "vm not provisioned"
def appliance_preupdate(temp_appliance_preconfig_funcscope_upgrade, appliance): '''Reconfigure appliance partitions and adds repo file for upgrade''' update_url = ('update_url_' + ''.join([i for i in get_stream(appliance.version) if i.isdigit()])) temp_appliance_preconfig_funcscope_upgrade.db.extend_partition() urls = process_url(cfme_data['basic_info'][update_url]) output = build_file(urls) with tempfile.NamedTemporaryFile('w') as f: f.write(output) f.flush() os.fsync(f.fileno()) temp_appliance_preconfig_funcscope_upgrade.ssh_client.put_file( f.name, '/etc/yum.repos.d/update.repo') return temp_appliance_preconfig_funcscope_upgrade
import slumber import requests import time from cfme.utils.conf import env from cfme.utils.providers import providers_data from cfme.utils.version import get_stream # regexen to match templates to streams and pull out the date # stream names must be slugified (alphanumeric, dashes, underscores only) # regex must include month and day, may include year # If year is unset, will be the most recent month/day (not in the future) stream_matchers = ( (get_stream('latest'), '^miq-nightly-(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.2'), r'^cfme-52.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.3'), r'^cfme-53.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.4'), r'^cfme-54.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.5'), r'^cfme-55.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.6'), r'^cfme-56.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.7'), r'^cfme-57.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.8'), r'^cfme-58.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.9'), r'^cfme-59.*-(?P<month>\d{2})(?P<day>\d{2})'), # Nightly builds have potentially multiple version streams bound to them so we # cannot use get_stream() ('upstream_stable', r'^miq-stable-(?P<release>gapri[-\w]*?)' # release name limit to 5 chars r'-(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'), ('upstream_euwe', r'^miq-stable-(?P<release>euwe[-\w]*?)' r'-(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'), ('upstream_fine', r'^miq-stable-(?P<release>fine[-\w]*?)'
import slumber import requests from lxml import html import time from cfme.utils.conf import env from cfme.utils.log import logger from cfme.utils.providers import providers_data from cfme.utils.version import get_stream # regexen to match templates to streams and pull out the date # stream names must be slugified (alphanumeric, dashes, underscores only) # regex must include month and day, may include year # If year is unset, will be the most recent month/day (not in the future) stream_matchers = ( (get_stream('latest'), '^miq-nightly-(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.2'), r'^cfme-52.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.3'), r'^cfme-53.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.4'), r'^cfme-54.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.5'), r'^cfme-55.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.6'), r'^cfme-56.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.7'), r'^cfme-57.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.8'), r'^cfme-58.*-(?P<month>\d{2})(?P<day>\d{2})'), (get_stream('5.9'), r'^cfme-59.*-(?P<month>\d{2})(?P<day>\d{2})'), ( 'upstream_stable', r'^miq-stable-(?P<release>gapri[-\w]*?)' # release name limit to 5 chars r'-(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'), ('upstream_euwe', r'^miq-stable-(?P<release>euwe[-\w]*?)' r'-(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'),
def ha_appliances_with_providers(ha_multiple_preupdate_appliances, app_creds): """Configure HA environment Appliance one configuring dedicated database, 'ap' launch appliance_console, '' clear info screen, '5' setup db, '1' Creates v2_key, '1' selects internal db, '1' use partition, 'y' create dedicated db, 'pwd' db password, 'pwd' confirm db password + wait and '' finish. Appliance two creating region in dedicated database, 'ap' launch appliance_console, '' clear info screen, '5' setup db, '2' fetch v2_key, 'app0_ip' appliance ip address, '' default user, 'pwd' appliance password, '' default v2_key location, '2' create region in external db, '0' db region number, 'y' confirm create region in external db 'app0_ip', '' ip and default port for dedicated db, '' use default db name, '' default username, 'pwd' db password, 'pwd' confirm db password + wait and '' finish. Appliance one configuring primary node for replication, 'ap' launch appliance_console, '' clear info screen, '6' configure db replication, '1' configure node as primary, '1' cluster node number set to 1, '' default dbname, '' default user, 'pwd' password, 'pwd' confirm password, 'app0_ip' primary appliance ip, confirm settings and wait to configure, '' finish. Appliance three configuring standby node for replication, 'ap' launch appliance_console, '' clear info screen, '6' configure db replication, '2' configure node as standby, '2' cluster node number set to 2, '' default dbname, '' default user, 'pwd' password, 'pwd' confirm password, 'app0_ip' primary appliance ip, app1_ip standby appliance ip, confirm settings and wait to configure finish, '' finish. Appliance two configuring automatic failover of database nodes, 'ap' launch appliance_console, '' clear info screen '9' configure application database failover monitor, '1' start failover monitor. wait 30 seconds for service to start '' finish. """ apps0, apps1, apps2 = ha_multiple_preupdate_appliances app0_ip = apps0.hostname app1_ip = apps1.hostname pwd = app_creds["password"] # Configure first appliance as dedicated database interaction = SSHExpect(apps0) interaction.send('ap') interaction.answer('Press any key to continue.', '', timeout=20) interaction.answer('Choose the advanced setting: ', '5' if apps0.version < '5.10' else '7') # Configure Database interaction.answer('Choose the encryption key: |1|', '1') interaction.answer('Choose the database operation: ', '1') # On 5.10, rhevm provider: # # database disk # # 1) /dev/sr0: 0 MB # 2) /dev/vdb: 4768 MB # 3) Don't partition the disk interaction.answer('Choose the database disk: |1| ', '1' if apps0.version < '5.10' else '2') # Should this appliance run as a standalone database server? interaction.answer(r'\? \(Y\/N\): |N| ', 'y') interaction.answer('Enter the database password on localhost: ', pwd) interaction.answer('Enter the database password again: ', pwd) # Configuration activated successfully. interaction.answer('Press any key to continue.', '', timeout=6 * 60) wait_for(lambda: apps0.db.is_dedicated_active, num_sec=4 * 60) # Configure EVM webui appliance with create region in dedicated database interaction = SSHExpect(apps2) interaction.send('ap') interaction.answer('Press any key to continue.', '', timeout=20) interaction.answer('Choose the advanced setting: ', '5' if apps2.version < '5.10' else '7') # Configure Database interaction.answer('Choose the encryption key: |1| ', '2') interaction.send(app0_ip) interaction.answer('Enter the appliance SSH login: |root| ', '') interaction.answer('Enter the appliance SSH password: '******'Enter the path of remote encryption key: |/var/www/miq/vmdb/certs/v2_key|', '') interaction.answer('Choose the database operation: ', '2') interaction.answer('Enter the database region number: ', '0') # WARNING: Creating a database region will destroy any existing data and # cannot be undone. interaction.answer(r'Are you sure you want to continue\? \(Y\/N\):', 'y') interaction.answer('Enter the database hostname or IP address: ', app0_ip) interaction.answer('Enter the port number: |5432| ', '') interaction.answer('Enter the name of the database on .*: |vmdb_production| ', '') interaction.answer('Enter the username: |root|', '') interaction.answer('Enter the database password on .*: ', pwd) # Configuration activated successfully. interaction.answer('Press any key to continue.', '', timeout=360) apps2.evmserverd.wait_for_running() apps2.wait_for_web_ui() # Configure primary replication node interaction = SSHExpect(apps0) interaction.send('ap') interaction.answer('Press any key to continue.', '', timeout=20) # 6/8 for Configure Database Replication interaction.answer('Choose the advanced setting: ', '6' if apps1.version < '5.10' else '8') interaction.answer('Choose the database replication operation: ', '1') interaction.answer('Enter the number uniquely identifying ' 'this node in the replication cluster: ', '1') interaction.answer('Enter the cluster database name: |vmdb_production| ', '') interaction.answer('Enter the cluster database username: |root| ', '') interaction.answer('Enter the cluster database password: '******'Enter the cluster database password: '******'Enter the primary database hostname or IP address: |.*| ', app0_ip) interaction.answer(r'Apply this Replication Server Configuration\? \(Y/N\): ', 'y') interaction.answer('Press any key to continue.', '') if BZ(1732092, forced_streams=get_stream(apps1.version)).blocks: assert apps1.ssh_client.run_command('setenforce 0').success # Configure secondary (standby) replication node interaction = SSHExpect(apps1) interaction.send('ap') interaction.answer('Press any key to continue.', '', timeout=20) interaction.answer('Choose the advanced setting: ', '6' if apps1.version < '5.10' else '8') # 6/8 for Configure Database Replication # Configure Server as Standby interaction.answer('Choose the database replication operation: ', '2') interaction.answer('Choose the encryption key: |1| ', '2') interaction.send(app0_ip) interaction.answer('Enter the appliance SSH login: |root| ', '') interaction.answer('Enter the appliance SSH password: '******'Enter the path of remote encryption key: |/var/www/miq/vmdb/certs/v2_key|', '') interaction.answer('Choose the standby database disk: |1| ', '1' if apps1.version < '5.10' else '2') # "Enter " ... is on line above. interaction.answer('.*the number uniquely identifying this ' 'node in the replication cluster: ', '2') interaction.answer('Enter the cluster database name: |vmdb_production| ', '') interaction.answer('Enter the cluster database username: |root| ', '') interaction.answer('Enter the cluster database password: '******'Enter the cluster database password: '******'Enter the primary database hostname or IP address: ', app0_ip) interaction.answer('Enter the Standby Server hostname or IP address: |.*|', app1_ip) interaction.answer(r'Configure Replication Manager \(repmgrd\) for automatic ' r'failover\? \(Y/N\): ', 'y') interaction.answer(r'Apply this Replication Server Configuration\? \(Y/N\): ', 'y') interaction.answer('Press any key to continue.', '', timeout=5 * 60) # Configure automatic failover on EVM appliance interaction = SSHExpect(apps2) interaction.send('ap') interaction.answer('Press any key to continue.', '', timeout=20) interaction.expect('Choose the advanced setting: ') with waiting_for_ha_monitor_started(apps2, app1_ip, timeout=300): # Configure Application Database Failover Monitor interaction.send('8' if apps2.version < '5.10' else '10') interaction.answer('Choose the failover monitor configuration: ', '1') # Failover Monitor Service configured successfully interaction.answer('Press any key to continue.', '') # Add infra/cloud providers and create db backup provider_app_crud(VMwareProvider, apps2).setup() provider_app_crud(EC2Provider, apps2).setup() return ha_multiple_preupdate_appliances
output = build_file(urls) with tempfile.NamedTemporaryFile('w') as f: f.write(output) f.flush() os.fsync(f.fileno()) temp_appliance_preconfig_funcscope_upgrade.ssh_client.put_file( f.name, '/etc/yum.repos.d/update.repo') return temp_appliance_preconfig_funcscope_upgrade @pytest.mark.ignore_stream('5.5', 'upstream') @pytest.mark.tier(2) @pytest.mark.uncollectif( lambda db_version: db_version >= version.current_version() or version.get_stream(db_version) == version.current_stream()) @pytest.mark.meta( blockers=[BZ(1354466, unblock=lambda db_url: 'ldap' not in db_url)]) def test_db_migrate(app_creds, temp_appliance_extended_db, db_url, db_version, db_desc): app = temp_appliance_extended_db # Download the database logger.info("Downloading database: {}".format(db_desc)) url_basename = os_path.basename(db_url) rc, out = app.ssh_client.run_command( 'curl -o "/tmp/{}" "{}"'.format(url_basename, db_url), timeout=30) assert rc == 0, "Failed to download database: {}".format(out) # The v2_key is potentially here v2key_url = os_path.join(os_path.dirname(db_url), "v2_key")
def temp_appliance_preconfig_funcscope_upgrade(appliance): stream = (int(''.join([i for i in get_stream(appliance.version) if i.isdigit()])) - 1) stream = "downstream-{}z".format(stream) with temp_appliances(preconfigured=True, stream=stream) as appliances: yield appliances[0]
urls = process_url(cfme_data['basic_info'][update_url]) output = build_file(urls) with tempfile.NamedTemporaryFile('w') as f: f.write(output) f.flush() os.fsync(f.fileno()) temp_appliance_preconfig_funcscope_upgrade.ssh_client.put_file( f.name, '/etc/yum.repos.d/update.repo') return temp_appliance_preconfig_funcscope_upgrade @pytest.mark.ignore_stream('5.5', 'upstream') @pytest.mark.tier(2) @pytest.mark.uncollectif( lambda db_version: db_version >= version.current_version( ) or version.get_stream(db_version) == version.current_stream()) @pytest.mark.meta( blockers=[BZ(1354466, unblock=lambda db_url: 'ldap' not in db_url)]) def test_db_migrate(app_creds, temp_appliance_extended_db, db_url, db_version, db_desc): app = temp_appliance_extended_db # Download the database logger.info("Downloading database: {}".format(db_desc)) url_basename = os_path.basename(db_url) # MBU Backup is to large for /tmp loc = "/tmp/" if db_desc != "MBU Backup" else "/" rc, out = app.ssh_client.run_command('curl -o "{}{}" "{}"'.format( loc, url_basename, db_url), timeout=30) assert rc == 0, "Failed to download database: {}".format(out)