def setup_replication_env(cfme_version, provider, lease, sprout_poolid, desc): lease_time = tot_time(lease) """Multi appliance setup with multi region and replication from remote to global""" required_app_count = 2 if sprout_poolid: sprout_client = SproutClient.from_config() if sprout_client.call_method('pool_exists', sprout_poolid): sprout_pool = sprout_client.call_method('request_check', sprout_poolid) if len(sprout_pool['appliances']) >= required_app_count: print("Processing pool...") apps = [] for app in sprout_pool['appliances']: apps.append(IPAppliance(app['ip_address'])) else: sys.exit("Pool does not meet the minimum size requirements!") else: sys.exit("Pool not found!") else: print("Provisioning appliances") apps, request_id = provision_appliances( count=required_app_count, cfme_version=cfme_version, provider=provider, lease_time=lease_time ) print("Appliance pool lease time is {}".format(lease)) sprout_client = SproutClient.from_config() sprout_client.set_pool_description(request_id, desc) print("Configuring replicated environment") ip0 = apps[0].hostname ip1 = apps[1].hostname opt = '5' if cfme_version >= "5.8" else '8' command_set0 = ('ap', '', opt, '1', '1', 'y', '1', 'n', '99', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) apps[0].wait_for_evm_service() apps[0].wait_for_web_ui() print("Global region appliance provisioned and configured {}".format(ip0)) command_set1 = ('ap', '', opt, '2', ip0, '', pwd, '', '1', 'y', '1', 'n', '1', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].wait_for_evm_service() apps[1].wait_for_web_ui() print("Remote region appliance provisioned and configured {}".format(ip1)) print("Setup - Replication on remote appliance") apps[1].set_pglogical_replication(replication_type=':remote') print("Setup - Replication on global appliance") apps[0].set_pglogical_replication(replication_type=':global') apps[0].add_pglogical_replication_subscription(apps[1].hostname) print("Done!")
def fqdn_appliance(appliance, preconfigured): sp = SproutClient.from_config() available_providers = set(sp.call_method('available_providers')) required_providers = set(cfme_data['fqdn_providers']) usable_providers = available_providers & required_providers version = appliance.version.vstring stream = get_stream(appliance.version) for provider in usable_providers: try: apps, pool_id = sp.provision_appliances( count=1, preconfigured=preconfigured, version=version, stream=stream, provider=provider ) break except Exception as e: logger.warning("Couldn't provision appliance with following error:") logger.warning("{}".format(e)) continue else: logger.error("Couldn't provision an appliance at all") raise SproutException('No provision available') yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def temp_appliances(count=1, preconfigured=True, lease_time=180, stream=None, provider_type=None): """ Provisions one or more appliances for testing Args: count: Number of appliances preconfigured: True if the appliance should be already configured, False otherwise lease_time: Lease time in minutes (3 hours by default) """ apps = [] request_id = None try: sprout_client = SproutClient.from_config() apps, request_id = sprout_client.provision_appliances( provider_type=provider_type, count=count, lease_time=lease_time, preconfigured=preconfigured, stream=stream) yield apps finally: for app in apps: app.ssh_client.close() if request_id: sprout_client.destroy_pool(request_id)
def appliance_preupdate(old_version, appliance): series = appliance.version.series() update_url = "update_url_{}".format(series.replace('.', '')) """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" usable = [] sp = SproutClient.from_config() available_versions = set(sp.call_method('available_cfme_versions')) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable.sort(reverse=True) try: apps, pool_id = sp.provision_appliances(count=1, preconfigured=True, lease_time=180, version=str(usable[0])) except Exception as e: logger.exception( "Couldn't provision appliance with following error:{}".format(e)) raise SproutException('No provision available') apps[0].db.extend_partition() urls = cfme_data["basic_info"][update_url] apps[0].ssh_client.run_command( "curl {} -o /etc/yum.repos.d/update.repo".format(urls)) logger.info( 'Appliance update.repo file: \n%s', apps[0].ssh_client.run_command( 'cat /etc/yum.repos.d/update.repo').output) yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def appliance_preupdate(old_version, appliance): series = appliance.version.series() update_url = "update_url_{}".format(series.replace('.', '')) """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" usable = [] sp = SproutClient.from_config() available_versions = set(sp.call_method('available_cfme_versions')) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable.sort(reverse=True) try: apps, pool_id = sp.provision_appliances(count=1, preconfigured=True, lease_time=180, version=str(usable[0])) except Exception as e: logger.exception("Couldn't provision appliance with following error:{}".format(e)) raise SproutException('No provision available') apps[0].db.extend_partition() urls = process_url(cfme_data['basic_info'][update_url]) output = build_file(urls) with tempfile.NamedTemporaryFile('w') as f: f.write(output) f.flush() os.fsync(f.fileno()) apps[0].ssh_client.put_file( f.name, '/etc/yum.repos.d/update.repo') yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def setup_distributed_env(cfme_version, provider, lease, desc): lease_time = tot_time(lease) """multi appliance single region configuration (distributed setup, 1st appliance has a local database and workers, 2nd appliance has workers pointing at 1st appliance)""" print("Provisioning and configuring distributed environment") apps, request_id = provision_appliances(count=2, cfme_version=cfme_version, provider=provider, lease_time=lease_time) sprout_client = SproutClient.from_config() sprout_client.set_pool_description(request_id, desc) opt = '5' if cfme_version >= "5.8" else '8' ip0 = apps[0].hostname ip1 = apps[1].hostname port = (ip0, '') if cfme_version >= "5.8" else (ip0,) command_set0 = ('ap', '', opt, '1', '1', 'y', '1', 'n', '1', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) apps[0].wait_for_evm_service() apps[0].wait_for_web_ui() print("VMDB appliance provisioned and configured {}".format(ip0)) command_set1 = ('ap', '', opt, '2', ip0, '', pwd, '', '3') + port + ('', '', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].wait_for_evm_service() apps[1].wait_for_web_ui() print("Non-VMDB appliance provisioned and configured {}".format(ip1)) print("Appliance pool lease time is {}".format(lease))
def fqdn_appliance(appliance, preconfigured): sp = SproutClient.from_config() available_providers = set(sp.call_method('available_providers')) required_providers = set(cfme_data['fqdn_providers']) usable_providers = available_providers & required_providers version = appliance.version.vstring stream = get_stream(appliance.version) for provider in usable_providers: try: apps, pool_id = sp.provision_appliances( count=1, preconfigured=preconfigured, version=version, stream=stream, provider=provider) break except Exception as e: logger.warning( "Couldn't provision appliance with following error:") logger.warning("{}".format(e)) continue else: logger.error("Couldn't provision an appliance at all") raise SproutException('No provision available') yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def setup_distributed_env(cfme_version, provider_type, provider, lease, desc): lease_time = tot_time(lease) provider_type = None if provider else provider_type """multi appliance single region configuration (distributed setup, 1st appliance has a local database and workers, 2nd appliance has workers pointing at 1st appliance)""" print("Provisioning and configuring distributed environment") apps, request_id = provision_appliances(count=2, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) sprout_client = SproutClient.from_config() sprout_client.set_pool_description(request_id, desc) opt = '5' if cfme_version >= "5.8" else '8' ip0 = apps[0].hostname ip1 = apps[1].hostname port = (ip0, '') if cfme_version >= "5.8" else (ip0, ) command_set0 = ('ap', '', opt, '1', '1', 'y', '1', 'n', '1', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) apps[0].evmserverd.wait_for_running() apps[0].wait_for_web_ui() print("VMDB appliance provisioned and configured {}".format(ip0)) command_set1 = ('ap', '', opt, '2', ip0, '', pwd, '', '3') + port + ('', '', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].evmserverd.wait_for_running() apps[1].wait_for_web_ui() print("Non-VMDB appliance provisioned and configured {}".format(ip1)) print("Appliance pool lease time is {}".format(lease))
def appliance_preupdate(old_version, appliance): series = appliance.version.series() update_url = "update_url_{}".format(series.replace('.', '')) """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" usable = [] sp = SproutClient.from_config() available_versions = set(sp.call_method('available_cfme_versions')) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable.sort(reverse=True) try: apps, pool_id = sp.provision_appliances(count=1, preconfigured=True, lease_time=180, version=str(usable[0])) except Exception as e: logger.exception("Couldn't provision appliance with following error:{}".format(e)) raise SproutException('No provision available') apps[0].db.extend_partition() urls = cfme_data["basic_info"][update_url] apps[0].ssh_client.run_command( "curl {} -o /etc/yum.repos.d/update.repo".format(urls) ) logger.info('Appliance update.repo file: \n%s', apps[0].ssh_client.run_command('cat /etc/yum.repos.d/update.repo').output) yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def appliance_preupdate(old_version, appliance): series = appliance.version.series() update_url = "update_url_{}".format(series.replace('.', '')) """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" usable = [] sp = SproutClient.from_config() available_versions = set(sp.call_method('available_cfme_versions')) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable.sort(reverse=True) try: apps, pool_id = sp.provision_appliances(count=1, preconfigured=True, lease_time=180, version=str(usable[0])) except Exception as e: logger.exception("Couldn't provision appliance with following error:{}".format(e)) raise SproutException('No provision available') apps[0].db.extend_partition() urls = process_url(conf.cfme_data['basic_info'][update_url]) output = build_file(urls) with tempfile.NamedTemporaryFile('w') as f: f.write(output) f.flush() os.fsync(f.fileno()) apps[0].ssh_client.put_file( f.name, '/etc/yum.repos.d/update.repo') yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def sprout_appliances(appliance, config, count=1, preconfigured=True, lease_time=120, stream=None, provider_type=None, version=None, **kwargs): """ Provisions one or more appliances for testing Args: appliance: appliance object, used for version/stream defaults config: pytestconfig object to lookup sprout_user_key count: Number of appliances preconfigured: True if the appliance should be already configured, False otherwise lease_time: Lease time in minutes (2 hours by default) stream: defaults to appliance stream provider_type: no default, sprout chooses, string type otherwise version: defaults to appliance version, string type otherwise """ sprout_client = SproutClient.from_config( sprout_user_key=config.option.sprout_user_key or None) # if version is passed and stream is not, don't default to appliance stream # if stream is passed and version is not, don't default to appliance version # basically, let stream/version work independently, and only default if neither are set req_version = version or (appliance.version.vstring if not stream else None) req_stream = stream or (appliance.version.stream() if not version else None) apps, request_id = sprout_client.provision_appliances( provider_type=provider_type, count=count, version=req_version, preconfigured=preconfigured, stream=req_stream, lease_time=lease_time, **kwargs) if config.getoption("update_appliance"): for app in apps: logger.info("Initiating appliance update on temp appliance ...") urls = config.getoption("update_urls") app.update_rhel(*urls, reboot=True) # Web UI not available on unconfigured appliances if preconfigured: logger.info( "Appliance update finished on temp appliance, waiting for UI ..." ) app.wait_for_web_ui() logger.info("Appliance update finished on temp appliance...") try: # Renew in half the lease time interval which is number of minutes. with periodic_call(lease_time * 60 / 2., sprout_client.prolong_pool, (request_id, lease_time)): yield apps finally: sprout_client.destroy_pool(request_id)
def provision_appliances(count, cfme_version, provider, lease_time): sprout_client = SproutClient.from_config() apps, request_id = sprout_client.provision_appliances( version=str(cfme_version), count=count, preconfigured=False, lease_time=lease_time, provider=provider) return apps, request_id
def fqdn_appliance(appliance, preconfigured, count): version = appliance.version.vstring stream = get_stream(appliance.version) try: sprout_client = SproutClient.from_config() apps, request_id = sprout_client.provision_appliances( provider_type='rhevm', count=count, version=version, preconfigured=preconfigured, stream=stream) yield apps finally: for app in apps: app.ssh_client.close() if request_id: sprout_client.destroy_pool(request_id)
def coverage_report_jenkins(jenkins_url, jenkins_jobs, jenkins_user, jenkins_token, appliance_ip, appliance_version, wave_size): """Aggregate coverage data from jenkins job(s) and upload to sonarqube""" if appliance_ip is None and appliance_version is None: ValueError('Must specify either --appliance-ip or --find-appliance') if appliance_ip is not None and appliance_version is not None: ValueError('--appliance-ip and --find-appliance are mutually exclusive options') # Find appliance using sprout if asked to do so: if appliance_version is not None: # TODO: Upstream support group = 'downstream-{}z'.format(''.join(appliance_version.split('.')[:2])) sprout = SproutClient.from_config() logger.info('requesting an appliance from sprout for %s/%s', group, appliance_version) pool_id = sprout.request_appliances( group, version=appliance_version, lease_time=env.sonarqube.scanner_lease) logger.info('Requested pool %s', pool_id) result = None try: while not result or not (result['fulfilled'] and result['finished']): result = sprout.request_check(pool_id) appliance_ip = result['appliances'][0]['ip_address'] logger.info('Received an appliance with IP address: %s', appliance_ip) with IPAppliance(hostname=appliance_ip) as appliance: exit(aggregate_coverage( appliance, jenkins_url, jenkins_user, jenkins_token, jenkins_jobs, wave_size)) finally: with diaper: sprout.destroy_pool(pool_id) else: # Use and existing appliance. with IPAppliance(hostname=appliance_ip) as appliance: exit(aggregate_coverage( appliance, jenkins_url, jenkins_user, jenkins_token, jenkins_jobs, wave_size))
def setup_ha_env(cfme_version, provider_type, provider, lease, desc): lease_time = tot_time(lease) provider_type = None if provider else provider_type """multi appliance setup consisting of dedicated primary and standy databases with a single UI appliance.""" print("Provisioning and configuring HA environment") apps, request_id = provision_appliances(count=3, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) sprout_client = SproutClient.from_config() sprout_client.set_pool_description(request_id, desc) ip0 = apps[0].hostname ip1 = apps[1].hostname ip2 = apps[2].hostname opt = '5' if cfme_version >= "5.8" else '8' rep = '6' if cfme_version >= "5.8" else '9' mon = VersionPicker({ Version.lowest(): '12', '5.8': '9', '5.9.3': '8' }).pick(cfme_version) port = (ip0, '') if cfme_version >= "5.8" else (ip0, ) command_set0 = ('ap', '', opt, '1', '1', '1', 'y', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) wait_for(lambda: apps[0].db.is_dedicated_active) print("Dedicated database provisioned and configured {}".format(ip0)) command_set1 = ('ap', '', opt, '1', '2', '1', 'y') + port + ('', '', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].wait_for_evm_service() apps[1].wait_for_web_ui() print("Non-VMDB appliance provisioned and region created {}".format(ip1)) command_set2 = ('ap', '', rep, '1', '1', '', '', pwd, pwd, ip0, 'y', '') apps[0].appliance_console.run_commands(command_set2) print("Primary HA node configured {}".format(ip0)) command_set3 = ('ap', '', rep, '2', '1', '2', '', '', pwd, pwd, ip0, ip2, 'y', TimedCommand('y', 300), '') apps[2].appliance_console.run_commands(command_set3) print("Secondary HA node provision and configured {}".format(ip2)) command_set4 = ('ap', '', mon, '1', '') apps[1].appliance_console.run_commands(command_set4) print("HA configuration complete") print("Appliance pool lease time is {}".format(lease))
def get_apps(requests, appliance, old_version, count, preconfigured, pytest_config): """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" series = appliance.version.series() update_url = "update_url_{}".format(series.replace(".", "")) usable = [] sp = SproutClient.from_config( sprout_user_key=pytest_config.option.sprout_user_key or None) available_versions = set(sp.call_method("available_cfme_versions")) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable_sorted = sorted(usable, key=lambda o: o.version) picked_version = usable_sorted[-1] apps = [] pool_id = None try: apps, pool_id = sp.provision_appliances( count=count, preconfigured=preconfigured, provider_type="rhevm", lease_time=180, version=str(picked_version), ) url = cfme_data["basic_info"][update_url] assert get_puddle_cfme_version(url) == appliance.version for app in apps: app.db.extend_partition() app.ssh_client.run_command( f"curl {url} -o /etc/yum.repos.d/update.repo") yield apps except AuthException: msg = ( 'Sprout credentials key or yaml maps missing or invalid,' 'unable to provision appliance version {}'.format(picked_version)) logger.exception(msg) pytest.skip(msg) finally: _collect_logs(pytest_config, apps) for app in apps: app.ssh_client.close() if pool_id: sp.destroy_pool(pool_id)
def setup_ha_env(cfme_version, provider_type, provider, lease, desc): lease_time = tot_time(lease) provider_type = None if provider else provider_type """multi appliance setup consisting of dedicated primary and standy databases with a single UI appliance.""" print("Provisioning and configuring HA environment") apps, request_id = provision_appliances(count=3, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) sprout_client = SproutClient.from_config() sprout_client.set_pool_description(request_id, desc) ip0 = apps[0].hostname ip1 = apps[1].hostname ip2 = apps[2].hostname port = (ip0, '') command_set0 = ('ap', '', '7', '1', '1', '2', 'y', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) wait_for(lambda: apps[0].db.is_dedicated_active) print(f"Dedicated database provisioned and configured {ip0}") command_set1 = ('ap', '', '7', '1', '2', '1', 'y') + port + ('', '', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].evmserverd.wait_for_running() apps[1].wait_for_web_ui() print(f"Non-VMDB appliance provisioned and region created {ip1}") command_set2 = ('ap', '', '8', '1', '1', '', '', pwd, pwd, ip0, 'y', '') apps[0].appliance_console.run_commands(command_set2) print(f"Primary HA node configured {ip0}") command_set3 = ('ap', '', '8', '2', '1', '2', '', '', pwd, pwd, ip0, ip2, 'y', TimedCommand('y', 300), '') apps[2].appliance_console.run_commands(command_set3) print(f"Secondary HA node provision and configured {ip2}") command_set4 = ('ap', '', '10', '1', '') apps[1].appliance_console.run_commands(command_set4) print("HA configuration complete") print(f"Appliance pool lease time is {lease}")
def sprout_appliances(appliance, config, count=1, preconfigured=True, lease_time=120, stream=None, provider_type=None, version=None, **kwargs): """ Provisions one or more appliances for testing Args: appliance: appliance object, used for version/stream defaults config: pytestconfig object to lookup sprout_user_key count: Number of appliances preconfigured: True if the appliance should be already configured, False otherwise lease_time: Lease time in minutes (3 hours by default) stream: defaults to appliance stream provider_type: no default, sprout chooses, string type otherwise version: defaults to appliance version, string type otherwise """ sprout_client = SproutClient.from_config( sprout_user_key=config.option.sprout_user_key or None) # if version is passed and stream is not, don't default to appliance stream # if stream is passed and version is not, don't default to appliance version # basically, let stream/version work independently, and only default if neither are set req_version = version or (appliance.version.vstring if not stream else None) req_stream = stream or (appliance.version.stream() if not version else None) apps, request_id = sprout_client.provision_appliances( provider_type=provider_type, count=count, version=req_version, preconfigured=preconfigured, stream=req_stream, lease_time=lease_time, **kwargs) try: yield apps finally: sprout_client.destroy_pool(request_id)
def setup_ha_env(cfme_version, provider_type, provider, lease, desc): lease_time = tot_time(lease) provider_type = None if provider else provider_type """multi appliance setup consisting of dedicated primary and standy databases with a single UI appliance.""" print("Provisioning and configuring HA environment") apps, request_id = provision_appliances(count=3, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) sprout_client = SproutClient.from_config() sprout_client.set_pool_description(request_id, desc) ip0 = apps[0].hostname ip1 = apps[1].hostname ip2 = apps[2].hostname opt = '5' if cfme_version >= "5.8" else '8' rep = '6' if cfme_version >= "5.8" else '9' mon = VersionPicker({ Version.lowest(): '12', '5.8': '9', '5.9.3': '8' }).pick(cfme_version) port = (ip0, '') if cfme_version >= "5.8" else (ip0,) command_set0 = ('ap', '', opt, '1', '1', '1', 'y', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) wait_for(lambda: apps[0].db.is_dedicated_active) print("Dedicated database provisioned and configured {}".format(ip0)) command_set1 = ('ap', '', opt, '1', '2', '1', 'y') + port + ('', '', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].evmserverd.wait_for_running() apps[1].wait_for_web_ui() print("Non-VMDB appliance provisioned and region created {}".format(ip1)) command_set2 = ('ap', '', rep, '1', '1', '', '', pwd, pwd, ip0, 'y', '') apps[0].appliance_console.run_commands(command_set2) print("Primary HA node configured {}".format(ip0)) command_set3 = ('ap', '', rep, '2', '1', '2', '', '', pwd, pwd, ip0, ip2, 'y', TimedCommand('y', 300), '') apps[2].appliance_console.run_commands(command_set3) print("Secondary HA node provision and configured {}".format(ip2)) command_set4 = ('ap', '', mon, '1', '') apps[1].appliance_console.run_commands(command_set4) print("HA configuration complete") print("Appliance pool lease time is {}".format(lease))
def sprout_appliances( appliance, config, count=1, preconfigured=True, lease_time=120, stream=None, provider_type=None, version=None, **kwargs ): """ Provisions one or more appliances for testing Args: appliance: appliance object, used for version/stream defaults config: pytestconfig object to lookup sprout_user_key count: Number of appliances preconfigured: True if the appliance should be already configured, False otherwise lease_time: Lease time in minutes (3 hours by default) stream: defaults to appliance stream provider_type: no default, sprout chooses, string type otherwise version: defaults to appliance version, string type otherwise """ sprout_client = SproutClient.from_config(sprout_user_key=config.option.sprout_user_key or None) # if version is passed and stream is not, don't default to appliance stream # if stream is passed and version is not, don't default to appliance version # basically, let stream/version work independently, and only default if neither are set req_version = version or (appliance.version.vstring if not stream else None) req_stream = stream or (appliance.version.stream() if not version else None) apps, request_id = sprout_client.provision_appliances( provider_type=provider_type, count=count, version=req_version, preconfigured=preconfigured, stream=req_stream, lease_time=lease_time, **kwargs ) try: yield apps finally: sprout_client.destroy_pool(request_id)
def temp_appliances(count=1, preconfigured=True, lease_time=180, stream=None, provider_type=None): """ Provisions one or more appliances for testing Args: count: Number of appliances preconfigured: True if the appliance should be already configured, False otherwise lease_time: Lease time in minutes (3 hours by default) """ apps = [] request_id = None try: sprout_client = SproutClient.from_config() apps, request_id = sprout_client.provision_appliances(provider_type=provider_type, count=count, lease_time=lease_time, preconfigured=preconfigured, stream=stream) yield apps finally: for app in apps: app.ssh_client.close() if request_id: sprout_client.destroy_pool(request_id)
def get_apps(appliance, old_version, count, preconfigured): """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" series = appliance.version.series() update_url = "update_url_{}".format(series.replace(".", "")) usable = [] sp = SproutClient.from_config() available_versions = set(sp.call_method("available_cfme_versions")) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable_sorted = sorted(usable, key=lambda o: o.version) apps = [] pool_id = None try: apps, pool_id = sp.provision_appliances( count=count, preconfigured=preconfigured, provider_type="rhevm", lease_time=180, version=str(usable_sorted[-1]), ) urls = cfme_data["basic_info"][update_url] for app in apps: app.db.extend_partition() app.ssh_client.run_command( "curl {} -o /etc/yum.repos.d/update.repo".format(urls)) yield apps except Exception as e: logger.error( "Couldn't provision appliance with following error:{}".format(e)) raise SproutException("Not able to configure provision request") finally: for app in apps: app.ssh_client.close() if pool_id: sp.destroy_pool(pool_id)
def appliance_preupdate(old_version, appliance, request): series = appliance.version.series() update_url = "update_url_{}".format(series.replace('.', '')) """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" usable = [] sprout = SproutClient.from_config(sprout_user_key=request.config.getoption( 'sprout_user_key', default=None) or None) available_versions = set(sprout.call_method('available_cfme_versions')) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable.sort(reverse=True) try: apps, pool_id = sprout.provision_appliances(count=1, preconfigured=True, lease_time=180, version=str(usable[0])) except AuthException: msg = ('Sprout credentials key or yaml maps missing or invalid,' 'unable to provision appliance version %s for preupdate'.format( str(usable[0]))) logger.exception(msg) pytest.skip(msg) apps[0].db.extend_partition() urls = cfme_data["basic_info"][update_url] apps[0].ssh_client.run_command( f"curl {urls} -o /etc/yum.repos.d/update.repo") logger.info( 'Appliance update.repo file: \n%s', apps[0].ssh_client.run_command( 'cat /etc/yum.repos.d/update.repo').output) yield apps[0] apps[0].ssh_client.close() sprout.destroy_pool(pool_id)
def get_apps(appliance, old_version, count, preconfigured): """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" series = appliance.version.series() update_url = "update_url_{}".format(series.replace(".", "")) usable = [] sp = SproutClient.from_config() available_versions = set(sp.call_method("available_cfme_versions")) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable_sorted = sorted(usable, key=lambda o: o.version) apps = [] pool_id = None try: apps, pool_id = sp.provision_appliances( count=count, preconfigured=preconfigured, provider_type="rhevm", lease_time=180, version=str(usable_sorted[-1]), ) urls = cfme_data["basic_info"][update_url] for app in apps: app.db.extend_partition() app.ssh_client.run_command( "curl {} -o /etc/yum.repos.d/update.repo".format(urls) ) yield apps except Exception as e: logger.error("Couldn't provision appliance with following error:{}".format(e)) raise SproutException("Not able to configure provision request") finally: for app in apps: app.ssh_client.close() if pool_id: sp.destroy_pool(pool_id)
def claim_appliance_and_aggregate(jenkins_url, jenkins_jobs, version, jenkins_user, jenkins_token, wave_size): # TODO: Upstream support group = 'downstream-' + ''.join(version.split('.')[:2]) + 'z' sprout = SproutClient.from_config() logger.info('requesting an appliance from sprout for %s/%s', group, version) pool_id = sprout.request_appliances(group, version=version, lease_time=env.sonarqube.scanner_lease) logger.info('Requested pool %s', pool_id) result = None try: while not result or not (result['fulfilled'] and result['finished']): result = sprout.request_check(pool_id) appliance_ip = result['appliances'][0]['ip_address'] logger.info('Received an appliance with IP address: %s', appliance_ip) with IPAppliance(hostname=appliance_ip) as appliance: exit( aggregate_coverage(appliance, jenkins_url, jenkins_user, jenkins_token, jenkins_jobs, wave_size)) finally: with diaper: sprout.destroy_pool(pool_id)
def appliance_preupdate(old_version, appliance): series = appliance.version.series() update_url = "update_url_{}".format(series.replace('.', '')) """Requests appliance from sprout based on old_versions, edits partitions and adds repo file for update""" usable = [] sp = SproutClient.from_config(sprout_user_key=pytest.config.option.sprout_user_key or None) available_versions = set(sp.call_method('available_cfme_versions')) for a in available_versions: if a.startswith(old_version): usable.append(Version(a)) usable.sort(reverse=True) try: apps, pool_id = sp.provision_appliances(count=1, preconfigured=True, lease_time=180, version=str(usable[0])) except AuthException: msg = ('Sprout credentials key or yaml maps missing or invalid,' 'unable to provision appliance version %s for preupdate'.format(str(usable[0]))) logger.exception(msg) pytest.skip(msg) apps[0].db.extend_partition() urls = cfme_data["basic_info"][update_url] apps[0].ssh_client.run_command( "curl {} -o /etc/yum.repos.d/update.repo".format(urls) ) logger.info('Appliance update.repo file: \n%s', apps[0].ssh_client.run_command('cat /etc/yum.repos.d/update.repo').output) yield apps[0] apps[0].ssh_client.close() sp.destroy_pool(pool_id)
def setup_multiregion_env(cfme_version, provider_type, provider, lease, sprout_poolid, desc, remote_nodes, add_prov): lease_time = tot_time(lease) provider_type = None if provider else provider_type """Multi appliance setup with multi region and replication from remote to global""" sprout_client = SproutClient.from_config() required_app_count = 1 # global app required_app_count += remote_nodes if sprout_poolid: if sprout_client.call_method('pool_exists', sprout_poolid): sprout_pool = sprout_client.call_method('request_check', sprout_poolid) if len(sprout_pool['appliances']) >= required_app_count: print("Processing pool...") apps = [] for app in sprout_pool['appliances']: apps.append(IPAppliance(app['ip_address'])) sprout_client.set_pool_description(sprout_poolid, desc) else: sys.exit("Pool does not meet the minimum size requirements!") else: sys.exit("Pool not found!") else: print("Provisioning appliances") apps, request_id = provision_appliances(count=required_app_count, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) print("Appliance pool lease time is {}".format(lease)) sprout_client.set_pool_description(request_id, desc) print("Appliances Provisioned") print("Configuring Replicated Environment") global_app = apps[0] gip = global_app.hostname remote_apps = apps[1:] print("Global Appliance Configuration") command_set0 = ('ap', '', '7', '1', '1', '2', 'n', '99', pwd, TimedCommand(pwd, 360), '') global_app.appliance_console.run_commands(command_set0) global_app.evmserverd.wait_for_running() global_app.wait_for_web_ui() print("Done: Global @ {}".format(gip)) for num, app in enumerate(remote_apps): region_n = str(num + 1 * 10) print("Remote Appliance Configuration") command_set1 = ('ap', '', '7', '2', gip, '', pwd, '', '1', '2', 'n', region_n, pwd, TimedCommand(pwd, 360), '') app.appliance_console.run_commands(command_set1) app.evmserverd.wait_for_running() app.wait_for_web_ui() print("Done: Remote @ {}, region: {}".format(app.hostname, region_n)) print("Configuring Replication") print("Setup - Replication on remote appliance") app.set_pglogical_replication(replication_type=':remote') print("Setup - Replication on global appliance") global_app.set_pglogical_replication(replication_type=':global') for app in remote_apps: global_app.add_pglogical_replication_subscription(app.hostname) random.shuffle(remote_apps) if add_prov: for app, prov_id in zip(cycle(remote_apps), add_prov): stack.push(app) prov = get_crud(prov_id) print("Adding provider {} to appliance {}".format( prov_id, app.hostname)) prov.create() stack.pop() print("Done!")
def setup_multiregion_env(cfme_version, provider_type, provider, lease, sprout_poolid, desc, remote_nodes, add_prov): lease_time = tot_time(lease) provider_type = None if provider else provider_type """Multi appliance setup with multi region and replication from remote to global""" sprout_client = SproutClient.from_config() required_app_count = 1 # global app required_app_count += remote_nodes if sprout_poolid: if sprout_client.call_method('pool_exists', sprout_poolid): sprout_pool = sprout_client.call_method('request_check', sprout_poolid) if len(sprout_pool['appliances']) >= required_app_count: print("Processing pool...") apps = [] for app in sprout_pool['appliances']: apps.append(IPAppliance(app['ip_address'])) sprout_client.set_pool_description(sprout_poolid, desc) else: sys.exit("Pool does not meet the minimum size requirements!") else: sys.exit("Pool not found!") else: print("Provisioning appliances") apps, request_id = provision_appliances(count=required_app_count, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) print("Appliance pool lease time is {}".format(lease)) sprout_client.set_pool_description(request_id, desc) print("Appliances Provisioned") print("Configuring Replicated Environment") global_app = apps[0] gip = global_app.hostname remote_apps = apps[1:] print("Global Appliance Configuration") app_creds = { "username": credentials["database"]["username"], "password": credentials["database"]["password"], "sshlogin": credentials["ssh"]["username"], "sshpass": credentials["ssh"]["password"], } app_params = dict(region=99, dbhostname='localhost', username=app_creds['username'], password=app_creds['password'], dbname='vmdb_production', dbdisk=global_app.unpartitioned_disks[0]) global_app.appliance_console_cli.configure_appliance_internal(**app_params) global_app.evmserverd.wait_for_running() global_app.wait_for_web_ui() print("Done: Global @ {}".format(gip)) for num, app in enumerate(remote_apps): region_n = str((num + 1) * 10) print("Remote Appliance Configuration") app_params = dict(region=region_n, dbhostname='localhost', username=app_creds['username'], password=app_creds['password'], dbname='vmdb_production', dbdisk=app.unpartitioned_disks[0], fetch_key=gip, sshlogin=app_creds['sshlogin'], sshpass=app_creds['sshpass']) app.appliance_console_cli.configure_appliance_internal_fetch_key( **app_params) app.evmserverd.wait_for_running() app.wait_for_web_ui() print("Done: Remote @ {}, region: {}".format(app.hostname, region_n)) print("Configuring Replication") print("Setup - Replication on remote appliance") app.set_pglogical_replication(replication_type=':remote') print("Setup - Replication on global appliance") global_app.set_pglogical_replication(replication_type=':global') for app in remote_apps: global_app.add_pglogical_replication_subscription(app.hostname) random.shuffle(remote_apps) if add_prov: for app, prov_id in zip(cycle(remote_apps), add_prov): stack.push(app) prov = get_crud(prov_id) print("Adding provider {} to appliance {}".format( prov_id, app.hostname)) prov.create_rest() stack.pop() print("Done!")
def setup_replication_env(cfme_version, provider_type, provider, lease, sprout_poolid, desc, remote_worker): lease_time = tot_time(lease) provider_type = None if provider else provider_type """Multi appliance setup with multi region and replication from remote to global""" required_app_count = 2 sprout_client = SproutClient.from_config() if remote_worker: required_app_count += 1 if sprout_poolid: if sprout_client.call_method('pool_exists', sprout_poolid): sprout_pool = sprout_client.call_method('request_check', sprout_poolid) if len(sprout_pool['appliances']) >= required_app_count: print("Processing pool...") apps = [] for app in sprout_pool['appliances']: apps.append(IPAppliance(app['ip_address'])) sprout_client.set_pool_description(sprout_poolid, desc) else: sys.exit("Pool does not meet the minimum size requirements!") else: sys.exit("Pool not found!") else: print("Provisioning appliances") apps, request_id = provision_appliances(count=required_app_count, cfme_version=cfme_version, provider_type=provider_type, provider=provider, lease_time=lease_time) print(f"Appliance pool lease time is {lease}") sprout_client.set_pool_description(request_id, desc) print("Appliances Provisioned") print("Configuring Replicated Environment") ip0 = apps[0].hostname ip1 = apps[1].hostname print("Global Appliance Configuration") opt = '5' if cfme_version >= "5.8" else '8' command_set0 = ('ap', '', opt, '1', '1', 'y', '1', 'n', '99', pwd, TimedCommand(pwd, 360), '') apps[0].appliance_console.run_commands(command_set0) apps[0].evmserverd.wait_for_running() apps[0].wait_for_web_ui() print(f"Done: Global @ {ip0}") print("Remote Appliance Configuration") command_set1 = ('ap', '', opt, '2', ip0, '', pwd, '', '1', 'y', '1', 'n', '1', pwd, TimedCommand(pwd, 360), '') apps[1].appliance_console.run_commands(command_set1) apps[1].evmserverd.wait_for_running() apps[1].wait_for_web_ui() print(f"Done: Remote @ {ip1}") if remote_worker: print("Remote Worker Appliance Configuration") ip2 = apps[2].hostname command_set2 = [ 'ap', '', opt, '2', ip1, '', pwd, '', '3', ip1, '', '', '', pwd, pwd ] apps[2].appliance_console.run_commands(command_set2) apps[2].evmserverd.wait_for_running() apps[2].wait_for_web_ui() print(f"Done: Remote Worker @ {ip2}") print("Configuring Replication") print("Setup - Replication on remote appliance") apps[1].set_pglogical_replication(replication_type=':remote') print("Setup - Replication on global appliance") apps[0].set_pglogical_replication(replication_type=':global') apps[0].add_pglogical_replication_subscription(apps[1].hostname) print("Done!")
def client(self): """Provide additional kwargs to from_config for auth passing""" return SproutClient.from_config(sprout_user_key=self.sprout_user_key)
# Note we get our logging setup from this load too. from coverage_report_jenkins import main as coverage_report_jenkins from coverage_report_jenkins import logger if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('jenkins_url') parser.add_argument('jenkins_job_name') parser.add_argument('version') parser.add_argument('--jenkins-user', default=None) parser.add_argument('--jenkins-token', default=None) args = parser.parse_args() # TODO: Upstream support group = 'downstream-' + ''.join(args.version.split('.')[:2]) + 'z' sprout = SproutClient.from_config() logger.info('requesting an appliance from sprout for %s/%s', group, args.version) pool_id = sprout.request_appliances(group, version=args.version, lease_time=env.sonarqube.scanner_lease) logger.info('Requested pool %s', pool_id) result = None try: while not result or not (result['fulfilled'] and result['finished']): result = sprout.request_check(pool_id) appliance_ip = result['appliances'][0]['ip_address'] logger.info('received an appliance with IP address: %s', appliance_ip) with IPAppliance(hostname=appliance_ip) as appliance: exit( coverage_report_jenkins(appliance, args.jenkins_url,
# Note we get our logging setup from this load too. from coverage_report_jenkins import main as coverage_report_jenkins from coverage_report_jenkins import logger if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('jenkins_url') parser.add_argument('jenkins_job_name') parser.add_argument('version') parser.add_argument('--jenkins-user', default=None) parser.add_argument('--jenkins-token', default=None) args = parser.parse_args() # TODO: Upstream support group = 'downstream-' + ''.join(args.version.split('.')[:2]) + 'z' sprout = SproutClient.from_config() logger.info('requesting an appliance from sprout for %s/%s', group, args.version) pool_id = sprout.request_appliances( group, version=args.version, lease_time=env.sonarqube.scanner_lease) logger.info('Requested pool %s', pool_id) result = None try: while not result or not (result['fulfilled'] and result['finished']): result = sprout.request_check(pool_id) appliance_ip = result['appliances'][0]['ip_address'] logger.info('received an appliance with IP address: %s', appliance_ip) with IPAppliance(hostname=appliance_ip) as appliance: exit( coverage_report_jenkins(
def provision_appliances(count, cfme_version, provider, lease_time): sprout_client = SproutClient.from_config() apps, request_id = sprout_client.provision_appliances(version=str(cfme_version), count=count, preconfigured=False, lease_time=lease_time, provider=provider) return apps, request_id