def module_user(request, module_target_sat, default_org, default_location): """Creates admin user with default org set to module org and shares that user for all tests in the same test module. User's login contains test module name as a prefix. :rtype: :class:`nailgun.entities.Organization` """ # take only "module" from "tests.foreman.virtwho.test_module" test_module_name = request.module.__name__.split('.')[-1].split('_', 1)[-1] login = f'{test_module_name}_{gen_string("alphanumeric")}' password = gen_string('alphanumeric') logger.debug('Creating session user %r', login) user = module_target_sat.api.User( admin=True, default_organization=default_org, default_location=default_location, description= f'created automatically by airgun for module "{test_module_name}"', login=login, password=password, ).create() user.password = password yield user try: logger.debug('Deleting session user %r', user.login) user.delete(synchronous=False) except HTTPError as err: logger.warning('Unable to delete session user: %s', str(err))
def default_url_on_new_port(oldport, newport): """Creates context where the default capsule is forwarded on a new port :param int oldport: Port to be forwarded. :param int newport: New port to be used to forward `oldport`. :return: A string containing the new capsule URL with port. :rtype: str """ domain = settings.server.hostname with ssh.get_connection() as connection: command = f'ncat -kl -p {newport} -c "ncat {domain} {oldport}"' logger.debug(f'Creating tunnel: {command}') transport = connection.get_transport() channel = transport.open_session() channel.get_pty() channel.exec_command(command) # if exit_status appears until command_timeout, throw error if channel.exit_status_ready(): if channel.recv_exit_status() != 0: stderr = '' while channel.recv_stderr_ready(): stderr += channel.recv_stderr(1) logger.debug(f'Tunnel failed: {stderr}') # Something failed, so raise an exception. raise CapsuleTunnelError(stderr) yield f'https://{domain}:{newport}'
def create_import_export_local_dir(default_sat): """Creates a local directory inside root_dir on satellite from where the templates will be imported from or exported to. Also copies example template to that directory for test operations Finally, Removes a local directory after test is completed as a teardown part. """ dir_name = gen_string('alpha') root_dir = FOREMAN_TEMPLATE_ROOT_DIR dir_path = f'{root_dir}/{dir_name}' # Creating the directory and set the write context result = default_sat.execute( f'mkdir -p {dir_path} && ' f'chown foreman -R {root_dir} && ' f'restorecon -R -v {root_dir} && ' f'chcon -t httpd_sys_rw_content_t {dir_path} -R') if result.status != 0: logger.debug(result.stdout) logger.debug(result.stderr) pytest.fail( f"Failed to create local dir or set SELinux context. Error output: {result.stderr}" ) # Copying the file to new directory to be modified by tests default_sat.execute(f'cp example_template.erb {dir_path}') yield dir_name, dir_path default_sat.execute(f'rm -rf {dir_path}')
def satellite_factory(): if settings.server.get('deploy_arguments'): logger.debug( f'Original deploy arguments for sat: {settings.server.deploy_arguments}' ) _resolve_deploy_args(settings.server.deploy_arguments) logger.debug( f'Resolved deploy arguments for sat: {settings.server.deploy_arguments}' ) def factory(retry_limit=3, delay=300, workflow=None, **broker_args): if settings.server.deploy_arguments: broker_args.update(settings.server.deploy_arguments) logger.debug(f'Updated broker args for sat: {broker_args}') vmb = VMBroker( host_classes={'host': Satellite}, workflow=workflow or settings.server.deploy_workflow, **broker_args, ) timeout = (1200 + delay) * retry_limit sat = wait_for(vmb.checkout, timeout=timeout, delay=delay, fail_condition=[]) return sat.out return factory
def get_tests(self, launch=None, **test_args): """Returns tests data customized by kwargs parameters. This is a main function that will be called to retrieve the tests data of a particular test status or/and defect_type :param str launch: Dict of a target launch to fetch test items for :param dict test_args: apply the given filters and their values to the search request :returns dict: All filtered tests dict based on params data keyed by test name and test properties as value, in format - ```{'test_name1':test1_properties_dict, 'test_name2':test2_properties_dict}``` """ params = { 'page.size': 50, 'page.sort': 'name', 'filter.eq.launchId': launch["id"], 'filter.ne.type': "SUITE", } # parse the test filter parameters and turn them into API filter parameters if test_args is None: test_args = {} if test_args.get('status'): params['filter.in.status'] = ','.join(test_args['status']).upper() if test_args.get('defect_types'): params['filter.in.issueType'] = ','.join([ ReportPortal.defect_types[t] for t in test_args['defect_types'] ]) if test_args.get('user'): params['filter.has.attributeKey'] = 'assignee' params['filter.has.attributeValue'] = test_args['user'] # send HTTP request to RP API, retrieve the paginated results and join them together page = 1 total_pages = 1 resp_tests = [] while page <= total_pages: logger.debug(page) params['page.page'] = page resp = requests.get( url=f'{self.api_url}/item', headers=self.headers, params=params, verify=False, ) resp.raise_for_status() resp_tests.extend(resp.json()['content']) total_pages = resp.json()['page']['totalPages'] page += 1 # Only select tests matching the supplied paths. This is a workaround for RP API limitation # - unable to combine multiple filters of a same type if test_args.get('paths'): resp_tests = [ test for test in resp_tests if any([ path for path in test_args['paths'] if path in test['name'] ]) ] return resp_tests
def _read_log(ch, pattern): """Read a first line from the given channel buffer and return the matching line""" # read lines until the buffer is empty for log_line in ch.stdout().splitlines(): logger.debug(f'foreman-tail: {log_line}') if re.search(pattern, log_line): return log_line else: return None
def default_url_on_new_port(oldport, newport): """Creates context where the default capsule is forwarded on a new port :param int oldport: Port to be forwarded. :param int newport: New port to be used to forward `oldport`. :return: A string containing the new capsule URL with port. :rtype: str """ domain = settings.server.hostname client = ssh.get_client() pre_ncat_procs = client.execute('pgrep ncat').stdout.splitlines() with client.session.shell() as channel: # if ncat isn't backgrounded, it prevents the channel from closing command = f'ncat -kl -p {newport} -c "ncat {domain} {oldport}" &' # broker 0.1.25 makes these debug messages redundant logger.debug(f'Creating tunnel: {command}') channel.send(command) post_ncat_procs = client.execute('pgrep ncat').stdout.splitlines() ncat_pid = set(post_ncat_procs).difference(set(pre_ncat_procs)) if not len(ncat_pid): stderr = channel.get_exit_status()[1] logger.debug(f'Tunnel failed: {stderr}') # Something failed, so raise an exception. raise CapsuleTunnelError(f'Starting ncat failed: {stderr}') forward_url = f'https://{domain}:{newport}' logger.debug(f'Yielding capsule forward port url: {forward_url}') try: yield forward_url finally: logger.debug(f'Killing ncat pid: {ncat_pid}') client.execute(f'kill {ncat_pid.pop()}')
def _get_test_collection(selectable_tests, items): """Returns the selected and deselected items""" # Select test item if its in failed tests else deselect logger.debug( 'Selecting/Deselecting tests based on latest launch test results.') selected = [] deselected = [] for item in items: test_item = f"{item.location[0]}.{item.location[2]}" if test_item in selectable_tests: selected.append(item) else: deselected.append(item) return selected, deselected
def get_host_sat_version(): """Fetches host's Satellite version through SSH :return: Satellite version :rtype: version """ commands = (_extract_sat_version(c) for c in (_SAT_6_2_VERSION_COMMAND, _SAT_6_1_VERSION_COMMAND)) for version, ssh_result in commands: if version != 'Not Available': logger.debug(f'Host Satellite version: {version}') return version logger.warning(f'Host Satellite version not available: {ssh_result!r}') return version
def pytest_collection_modifyitems(items, config): """ Collects and modifies tests collection based on pytest options to select tests marked as failed/skipped and user specific tests in Report Portal """ fail_args = config.getoption('only_failed', False) skip_arg = config.getoption('only_skipped', False) user_arg = config.getoption('user', False) upgrades_rerun = config.getoption('upgrades_rerun', False) if not any([fail_args, skip_arg, user_arg, upgrades_rerun]): return rp = ReportPortal() version = settings.server.version sat_version = f'{version.base_version}.{version.epoch}' logger.info( f'Fetching Report Portal launches for target Satellite version: {sat_version}' ) launch = next( iter( rp.launches(sat_version=sat_version, launch_type='upgrades' if upgrades_rerun else 'satellite6').values())) _validate_launch(launch, sat_version) test_args = {} test_args.setdefault('status', list()) if fail_args: test_args['status'].append('failed') if not fail_args == 'all': defect_types = fail_args.split(',') if ',' in fail_args else [ fail_args ] allowed_args = [*rp.defect_types.keys()] if not set(defect_types).issubset(set(allowed_args)): raise pytest.UsageError( 'Incorrect values to pytest option \'--only-failed\' are provided as ' f'\'{fail_args}\'. It should be none/one/mix of {allowed_args}' ) test_args['defect_types'] = defect_types if skip_arg: test_args['status'].append('skipped') if user_arg: test_args['user'] = user_arg rp_tests = _get_tests(launch, **test_args) selected, deselected = _get_test_collection(rp_tests, items) logger.debug( f'Selected {len(selected)} and deselected {len(deselected)} tests based on latest ' 'launch test results.') config.hook.pytest_deselected(items=deselected) items[:] = selected
def get_host_os_version(): """Fetches host's OS version through SSH :return: str with version """ cmd = ssh.command('cat /etc/redhat-release') if cmd.stdout: version_description = cmd.stdout[0] version_re = r'Red Hat Enterprise Linux Server release (?P<version>\d(\.\d)*)' result = re.search(version_re, version_description) if result: host_os_version = f'RHEL{result.group("version")}' logger.debug(f'Host version: {host_os_version}') return host_os_version logger.warning(f'Host version not available: {cmd!r}') return 'Not Available'
def _read_log(ch, pattern): """Read a first line from the given channel buffer and return the matching line""" # read lines until the buffer is empty while ch.recv_ready(): log_line = '' # read bytes one-by-one until we have a complete log line while ch.recv_ready(): char = ch.recv(1).decode('utf-8') if char == '\n': break else: log_line += char logger.debug(f'foreman-tail: {log_line}') if re.search(pattern, log_line): return log_line else: return None
def get_connection( hostname=None, username=None, password=None, key_filename=None, key_string=None, timeout=None, port=22, ): """Yield an ssh connection object. The connection will be configured with the specified arguments or will fall-back to server configuration in the configuration file. Yield this SSH connection. The connection is automatically closed when the caller is done using it using ``contextlib``, so clients should use the ``with`` statement to handle the object:: with get_connection() as connection: ... kwargs are passed through to get_client :return: An SSH connection. :rtype: ``paramiko.SSHClient`` """ client = get_client( hostname=hostname, username=username, password=password, key_filename=key_filename, key_string=key_string, timeout=timeout, port=port, ) try: logger.debug(f'Instantiated Paramiko client {client._id}') logger.info('Connected to [%s]', hostname) yield client finally: client.close() logger.debug(f'Destroyed Paramiko client {client._id}')
def test_positive_export_all_templates_to_repo(self, module_org, git_repository, git_branch, url): """Assure all templates are exported if no filter is specified. :id: 0bf6fe77-01a3-4843-86d6-22db5b8adf3b :Steps: 1. Using nailgun export all templates to repository (ensure filters are empty) :expectedresults: 1. Assert all existing templates were exported to repository :BZ: 1785613 :parametrized: yes :CaseImportance: Low """ output = entities.Template().exports( data={ 'repo': f'{url}/{git.username}/{git_repository["name"]}', 'branch': git_branch, 'organization_ids': [module_org.id], }) auth = (git.username, git.password) api_url = f'http://{git.hostname}:{git.http_port}/api/v1/repos/{git.username}' res = requests.get( url=f'{api_url}/{git_repository["name"]}/git/trees/{git_branch}', auth=auth, params={'recursive': True}, ) res.raise_for_status() try: tree = json.loads(res.text)['tree'] except json.decoder.JSONDecodeError: logger.debug(res.json()) pytest.fail( f"Failed to parse output from git. Response: '{res.text}'") git_count = [row['path'].endswith('.erb') for row in tree].count(True) assert len(output['message']['templates']) == git_count
def _versions(self): """Sets satellite and snap version attributes of a launch""" version_compiler = re.compile(r'([\d\.]+)[\.-](\d+\.\d|[A-Z]+)') if not self.info['attributes']: logger.debug('Launch with no launch_attributes is detected. ' 'This will be removed from launch collection.') return launch_attrs = [ self.info['attributes'][attr]['value'] for attr in range(len(self.info['attributes'])) ] try: launch_name = next(filter(version_compiler.search, launch_attrs)) except StopIteration: logger.debug( 'Launch with no build name in launch_attributes is detected. ' f'The launch has tags {launch_attrs}') return self.satellite_version = re.search(version_compiler, launch_name).group(1) self.snap_version = re.search(version_compiler, launch_name).group(2)
def test_positive_run_receptor_installer(self, default_sat, subscribe_satellite, fixture_enable_receptor_repos): """Run Receptor installer ("Configure Cloud Connector") :CaseComponent: RHCloud-CloudConnector :Assignee: lhellebr :id: 811c7747-bec6-1a2d-8e5c-b5045d3fbc0d :expectedresults: The job passes, installs Receptor that peers with c.r.c :BZ: 1818076 """ result = default_sat.execute('stat /etc/receptor/*/receptor.conf') if result.status == 0: pytest.skip( 'Cloud Connector has already been configured on this system. ' 'It is possible to reconfigure it but then the test would not really ' 'check if everything is correctly configured from scratch. Skipping.' ) # Copy foreman-proxy user's key to root@localhost user's authorized_keys default_sat.add_rex_key(satellite=default_sat) # Set Host parameter source_display_name to something random. # To avoid 'name has already been taken' error when run multiple times # on a machine with the same hostname. host_id = Host.info({'name': default_sat.hostname})['id'] Host.set_parameter({ 'host-id': host_id, 'name': 'source_display_name', 'value': gen_string('alpha') }) template_name = 'Configure Cloud Connector' invocation = make_job_invocation({ 'async': True, 'job-template': template_name, 'inputs': f'satellite_user="******",\ satellite_password="******"', 'search-query': f'name ~ {default_sat.hostname}', }) invocation_id = invocation['id'] wait_for( lambda: entities.JobInvocation(id=invocation_id).read(). status_label in ['succeeded', 'failed'], timeout='1500s', ) result = JobInvocation.get_output({ 'id': invocation_id, 'host': default_sat.hostname }) logger.debug( f'Invocation output>>\n{result}\n<<End of invocation output') # if installation fails, it's often due to missing rhscl repo -> print enabled repos repolist = default_sat.execute('yum repolist') logger.debug(f'Repolist>>\n{repolist}\n<<End of repolist') assert entities.JobInvocation(id=invocation_id).read().status == 0 assert 'project-receptor.satellite_receptor_installer' in result assert 'Exit status: 0' in result # check that there is one receptor conf file and it's only readable # by the receptor user and root result = default_sat.execute( 'stat /etc/receptor/*/receptor.conf --format "%a:%U"') assert all(filestats == '400:foreman-proxy' for filestats in result.stdout.strip().split('\n')) result = default_sat.execute( 'ls -l /etc/receptor/*/receptor.conf | wc -l') assert int(result.stdout.strip()) >= 1
def get_data_bz(bz_numbers, cached_data=None): # pragma: no cover """Get a list of marked BZ data and query Bugzilla REST API. Arguments: bz_numbers {list of str} -- ['123456', ...] cached_data Returns: [list of dicts] -- [{'id':..., 'status':..., 'resolution': ...}] """ if not bz_numbers: return [] cached_by_call = CACHED_RESPONSES['get_data'].get(str(sorted(bz_numbers))) if cached_by_call: return cached_by_call if cached_data: logger.debug(f"Using cached data for {set(bz_numbers)}") if not all([f'BZ:{number}' in cached_data for number in bz_numbers]): logger.debug("There are BZs out of cache.") return [item['data'] for _, item in cached_data.items() if 'data' in item] # Ensure API key is set if not settings.bugzilla.api_key: logger.warning( "Config file is missing bugzilla api_key " "so all tests with skip_if_open mark is skipped. " "Provide api_key or a bz_cache.json." ) # Provide default data for collected BZs return [get_default_bz(number) for number in bz_numbers] # No cached data so Call Bugzilla API logger.debug(f"Calling Bugzilla API for {set(bz_numbers)}") bz_fields = [ "id", "summary", "status", "resolution", "cf_last_closed", "last_change_time", "creation_time", "flags", "keywords", "dupe_of", "target_milestone", "cf_clone_of", "clone_ids", "depends_on", ] # Following fields are dynamically calculated/loaded for field in ('is_open', 'clones', 'version'): assert field not in bz_fields response = requests.get( f"{settings.bugzilla.url}/rest/bug", params={ "id": ",".join(set(bz_numbers)), "include_fields": ",".join(bz_fields), }, headers={"Authorization": f"Bearer {settings.bugzilla.api_key}"}, ) response.raise_for_status() data = response.json().get('bugs') CACHED_RESPONSES['get_data'][str(sorted(bz_numbers))] = data return data
def pytest_collection_modifyitems(items, config): """ Collects and modifies test collection based on the pytest options to select the tests marked as failed/skipped and user-specific tests in Report Portal """ rp_url = settings.report_portal.portal_url or config.getini('rp_endpoint') rp_uuid = config.getini('rp_uuid') or settings.report_portal.api_key # prefer dynaconf setting before ini config as pytest-reportportal plugin uses default value # for `rp_launch` if none is set there rp_launch_name = settings.report_portal.launch_name or config.getini('rp_launch') rp_project = config.getini('rp_project') or settings.report_portal.project fail_args = config.getoption('only_failed', False) skip_arg = config.getoption('only_skipped', False) user_arg = config.getoption('user', False) ref_launch_uuid = config.getoption('rp_reference_launch_uuid', None) or config.getoption( 'rp_rerun_of', None ) tests = [] if not any([fail_args, skip_arg, user_arg]): return rp = ReportPortal(rp_url=rp_url, rp_api_key=rp_uuid, rp_project=rp_project) if ref_launch_uuid: logger.info(f'Fetching A reference Report Portal launch {ref_launch_uuid}') ref_launches = rp.get_launches(uuid=ref_launch_uuid) if not ref_launches: raise LaunchError( f'Provided reference launch {ref_launch_uuid} was not found or is not finished' ) else: sat_release = get_sat_version().base_version sat_snap = settings.server.version.get('snap', '') if not all([sat_release, sat_snap, (len(sat_release.split('.')) == 3)]): raise pytest.UsageError( '--failed|skipped-only requires a reference launch id or' ' a full satellite version (x.y.z-a.b) to be provided.' f' sat_release: {sat_release}, sat_snap: {sat_snap} were provided instead' ) sat_version = f'{sat_release}-{sat_snap}' logger.info( f'Fetching A reference Report Portal launch by Satellite version: {sat_version}' ) ref_launches = rp.get_launches(name=rp_launch_name, sat_version=sat_version) if not ref_launches: raise LaunchError( f'No suitable Report portal launches for name: {rp_launch_name}' f' and version: {sat_version} found' ) test_args = {} test_args.setdefault('status', list()) if skip_arg: test_args['status'].append('SKIPPED') if fail_args: test_args['status'].append('FAILED') if not fail_args == 'all': defect_types = fail_args.split(',') allowed_args = [*rp.defect_types.keys()] if not set(defect_types).issubset(set(allowed_args)): raise pytest.UsageError( 'Incorrect values to pytest option \'--only-failed\' are provided as ' f'\'{fail_args}\'. It should be none/one/mix of {allowed_args}' ) test_args['defect_types'] = defect_types if user_arg: test_args['user'] = user_arg test_args['paths'] = config.args for ref_launch in ref_launches: _validate_launch(ref_launch) tests.extend(rp.get_tests(launch=ref_launch, **test_args)) # remove inapplicable tests from the current test collection deselected = [ i for i in items if f'{i.location[0]}.{i.location[2]}'.replace('::', '.') not in [t['name'].replace('::', '.') for t in tests] ] selected = list(set(items) - set(deselected)) logger.debug( f'Selected {len(selected)} and deselected {len(deselected)} tests based on latest/given-/ ' 'launch test results.' ) config.hook.pytest_deselected(items=deselected) items[:] = selected
def log_version_info(msg, template): logger.debug(template, func.__name__, func.__module__, msg)
def test_positive_configure_cloud_connector(session, default_sat, subscribe_satellite, fixture_enable_receptor_repos): """Install Cloud Connector through WebUI button :id: 67e45cfe-31bb-51a8-b88f-27918c68f32e :Steps: 1. Navigate to Configure > Inventory Upload 2. Click Configure Cloud Connector 3. Open the started job and wait until it is finished :expectedresults: The Cloud Connector has been installed and the service is running :CaseLevel: Integration :CaseComponent: RHCloud-CloudConnector :CaseImportance: Medium :assignee: lhellebr :BZ: 1818076 """ # Copy foreman-proxy user's key to root@localhost user's authorized_keys default_sat.add_rex_key(satellite=default_sat) # Set Host parameter source_display_name to something random. # To avoid 'name has already been taken' error when run multiple times # on a machine with the same hostname. host_id = Host.info({'name': default_sat.hostname})['id'] Host.set_parameter({ 'host-id': host_id, 'name': 'source_display_name', 'value': gen_string('alpha') }) with session: if session.cloudinventory.is_cloud_connector_configured(): pytest.skip( 'Cloud Connector has already been configured on this system. ' 'It is possible to reconfigure it but then the test would not really ' 'check if everything is correctly configured from scratch. Skipping.' ) session.cloudinventory.configure_cloud_connector() template_name = 'Configure Cloud Connector' invocation_id = (entities.JobInvocation().search( query={'search': f'description="{template_name}"'})[0].id) wait_for( lambda: entities.JobInvocation(id=invocation_id).read().status_label in ["succeeded", "failed"], timeout="1500s", ) result = JobInvocation.get_output({ 'id': invocation_id, 'host': default_sat.hostname }) logger.debug(f"Invocation output>>\n{result}\n<<End of invocation output") # if installation fails, it's often due to missing rhscl repo -> print enabled repos repolist = default_sat.execute('yum repolist') logger.debug(f"Repolist>>\n{repolist}\n<<End of repolist") assert entities.JobInvocation(id=invocation_id).read().status == 0 assert 'project-receptor.satellite_receptor_installer' in result assert 'Exit status: 0' in result # check that there is one receptor conf file and it's only readable # by the receptor user and root result = default_sat.execute( 'stat /etc/receptor/*/receptor.conf --format "%a:%U"') assert all(filestats == '400:foreman-proxy' for filestats in result.stdout.strip().split('\n')) result = default_sat.execute('ls -l /etc/receptor/*/receptor.conf | wc -l') assert int(result.stdout.strip()) >= 1