def pytest_collection_modifyitems(session, config, items): """Remove/Include stubbed tests from collection based on CLI option""" opt_passed = config.getvalue('mark_manuals_passed') opt_skipped = config.getvalue('mark_manuals_skipped') # TODO turn this into a flag or a choice option, this logic is just silly. mark_skipped = opt_skipped and not opt_passed include_stubbed = config.getvalue('include_stubbed') selected = [] deselected = [] for item in items: stub_marked = item.get_closest_marker(name='stubbed') # The test case is stubbed, and --include-stubbed was passed, include in collection if stub_marked and include_stubbed: selected.append(item) # enforce skip/pass behavior by marking skip if mark_skipped: logger.debug( f'Marking collected stubbed test "{item.nodeid}" to skip') item.add_marker(marker=pytest.mark.skip( reason=NOT_IMPLEMENTED)) continue # The test case is stubbed, but --include-stubbed was NOT passed, deselect the item if stub_marked and not include_stubbed: logger.debug(f'Deselecting stubbed test {item.nodeid}, ' 'use --include-stubbed to include in collection') deselected.append(item) continue # Its a non-stubbed item, this hook doesn't apply selected.append(item) config.hook.pytest_deselected(items=deselected) items[:] = selected
def pytest_collection_modifyitems(items, config): """ Collects and modifies tests collection based on pytest option to deselect tests for new infra """ include_onprem_provision = config.getoption('include_onprem_provisioning', False) include_libvirt = config.getoption('include_libvirt', False) include_eauth = config.getoption('include_external_auth', False) include_vlan = config.getoption('include_vlan_networking', False) include_non_satci_tests = config.getvalue('include_non_satci_tests').split( ',') selected = [] deselected = [] # Cloud Provisioning Test can be run on new pipeline for item in items: # Include/Exclude tests those are not part of SatQE CI item_component = item.get_closest_marker('component') if item_component and (item_component.args[0] in non_satCI_components): if item_component.args[0] in include_non_satci_tests: selected.append(item) else: deselected.append(item) continue item_marks = [m.name for m in item.iter_markers()] # Include / Exclude On Premises Provisioning Tests if 'on_premises_provisioning' in item_marks: selected.append( item) if include_onprem_provision else deselected.append(item) continue # Include / Exclude External Libvirt based Tests if 'libvirt_discovery' in item_marks: selected.append(item) if include_libvirt else deselected.append( item) continue # Include / Exclude External Auth based Tests if 'external_auth' in item_marks: selected.append(item) if include_eauth else deselected.append(item) continue # Include / Exclude VLAN networking based based Tests if 'vlan_networking' in item_marks: selected.append(item) if include_vlan else deselected.append(item) continue # This Plugin does not applies to this test selected.append(item) logger.debug( f'Selected {len(selected)} and deselected {len(deselected)} ' 'tests based on auto un-collectable markers and pytest options.') items[:] = selected or items config.hook.pytest_deselected(items=deselected)
def pytest_collection_modifyitems(session, items, config): """Generate the issue collection (using bz cache via bugzilla issue handler util) This collection includes pre-processed `is_open` status for each issue """ # generate_issue_collection will save a file, set by --bz-cache value pytest.issue_data = generate_issue_collection(items, config) # Add skipif markers, and modify collection based on --bz option bz_filters = config.getoption('BZ', None) if bz_filters: bz_filters = bz_filters.split(',') selected = [] deselected = [] for item in items: # Add a skipif marker for the issues skip_if_open = item.get_closest_marker('skip_if_open') if skip_if_open: # marker must have `BZ:123456` as argument. issue = skip_if_open.kwargs.get('reason') or skip_if_open.args[0] item.add_marker(pytest.mark.skipif(is_open(issue), reason=issue)) # remove items from collection if bz_filters: # Only include items which have BZ mark that includes any of the filtered bz numbers item_bz_marks = set(getattr(item.get_closest_marker('BZ', None), 'args', [])) if bool(set(bz_filters) & item_bz_marks): selected.append(item) else: logger.debug( f'Deselected test [{item.nodeid}] due to BZ filter {bz_filters} ' f'and available marks {item_bz_marks}' ) deselected.append(item) continue # move to the next item # append only when there is a BZ in common between the lists else: selected.append(item) config.hook.pytest_deselected(items=deselected) items[:] = selected
def pytest_collection_modifyitems(items, config): """ Collects and modifies tests collection based on pytest option to deselect tests for new infra """ include_onprem_provision = config.getoption('include_onprem_provisioning', False) include_libvirt = config.getoption('include_libvirt', False) include_eauth = config.getoption('include_external_auth', False) include_vlan = config.getoption('include_vlan_networking', False) selected = [] deselected = [] # Cloud Provisioning Test can be run on new pipeline for item in items: item_marks = [m.name for m in item.iter_markers()] # Include / Exclude On Premises Provisioning Tests if 'on_premises_provisioning' in item_marks: selected.append(item) if include_onprem_provision else deselected.append(item) continue # Include / Exclude External Libvirt based Tests if any(marker in item_marks for marker in ['libvirt_discovery', 'libvirt_content_host']): selected.append(item) if include_libvirt else deselected.append(item) continue # Include / Exclude External Auth based Tests if 'external_auth' in item_marks: selected.append(item) if include_eauth else deselected.append(item) continue # Include / Exclude VLAN networking based based Tests if 'vlan_networking' in item_marks: selected.append(item) if include_vlan else deselected.append(item) continue # This Plugin does not applies to this test selected.append(item) logger.debug( f'Selected {len(selected)} and deselected {len(deselected)} ' 'tests based on new infra markers.' ) items[:] = selected or items config.hook.pytest_deselected(items=deselected)
def pytest_collection_modifyitems(session, items, config): """Called after collection has been performed, may filter or re-order the items in-place. """ collection_logger.debug(f'Collected {len(items)} test cases') # Modify items based on collected issue_data deselected_items = [] for item in items: # 1. Deselect tests marked with @pytest.mark.deselect # WONTFIX BZs makes test to be dynamically marked as deselect. deselect = item.get_closest_marker('deselect') if deselect: deselected_items.append(item) reason = deselect.kwargs.get('reason', deselect.args) collection_logger.debug( f'Deselected test "{item.name}" reason: {reason}') # Do nothing more with deselected tests continue config.hook.pytest_deselected(items=deselected_items) items[:] = [item for item in items if item not in deselected_items]
def pytest_collection_modifyitems(session, items, config): """Add markers and user_properties for testimony token metadata user_properties is used by the junit plugin, and thus by many test report systems Handle test function/class/module/session scope metadata coming from test docblocks Apply user_properties, ibutsu metadata, and pytest markers Markers for metadata use the testimony token name as the mark name The value of the token for the mark is the first mark arg Control test collection for custom options related to testimony metadata """ # get RHEL version of the satellite rhel_version = get_sat_rhel_version().base_version sat_version = get_sat_version().base_version snap_version = settings.server.version.get('snap', '') # split the option string and handle no option, single option, multiple # config.getoption(default) doesn't work like you think it does, hence or '' importance = [ i for i in (config.getoption('importance') or '').split(',') if i != '' ] component = [ c for c in (config.getoption('component') or '').split(',') if c != '' ] assignee = [ a for a in (config.getoption('assignee') or '').split(',') if a != '' ] selected = [] deselected = [] logger.info('Processing test items to add testimony token markers') for item in items: if item.nodeid.startswith( 'tests/robottelo/') and 'test_junit' not in item.nodeid: # Unit test, no testimony markers continue # apply the marks for importance, component, and assignee # Find matches from docstrings starting at smallest scope item_docstrings = [ d for d in map(inspect.getdoc, (item.function, getattr(item, 'cls', None), item.module)) if d is not None ] item_mark_names = [m.name for m in item.iter_markers()] for docstring in item_docstrings: # Add marker starting at smallest docstring scope # only add the mark if it hasn't already been applied at a lower scope doc_component = component_regex.findall(docstring) if doc_component and 'component' not in item_mark_names: item.add_marker(pytest.mark.component(doc_component[0])) doc_importance = importance_regex.findall(docstring) if doc_importance and 'importance' not in item_mark_names: item.add_marker(pytest.mark.importance(doc_importance[0])) doc_assignee = assignee_regex.findall(docstring) if doc_assignee and 'assignee' not in item_mark_names: item.add_marker(pytest.mark.assignee(doc_assignee[0])) # add markers as user_properties so they are recorded in XML properties of the report for marker in item.iter_markers(): item.user_properties.append( (marker.name, next(iter(marker.args), None))) item.user_properties.append(("BaseOS", rhel_version)) item.user_properties.append(("SatelliteVersion", sat_version)) item.user_properties.append(("SnapVersion", snap_version)) item.user_properties.append( ("start_time", datetime.datetime.utcnow().strftime(FMT_XUNIT_TIME))) # add custom ibutsu metadata fields for test case grouping and heatmaps if hasattr(item, "_ibutsu"): item._ibutsu["data"]["metadata"].update({ # TODO Work with ibutsu team, better mechanism for defining 'special' data # TODO Add sat version to this item data at test execution time "component": item.get_closest_marker('component').args[0] }) # exit early if no filters were passed if importance or component or assignee: # Filter test collection based on CLI options for filtering # filters should be applied together # such that --component Repository --importance Critical --assignee jsmith # only collects tests which have all three of these marks # https://github.com/pytest-dev/pytest/issues/1373 Will make this way easier # testimony requires both importance and component, this will blow up if its forgotten importance_marker = item.get_closest_marker('importance').args[0] if importance and importance_marker not in importance: logger.debug( f'Deselected test {item.nodeid} due to "--importance {importance}",' f'test has importance mark: {importance_marker}') deselected.append(item) continue component_marker = item.get_closest_marker('component').args[0] if component and component_marker not in component: logger.debug( f'Deselected test {item.nodeid} due to "--component {component}",' f'test has component mark: {component_marker}') deselected.append(item) continue assignee_marker = item.get_closest_marker('assignee').args[0] if assignee and assignee_marker not in assignee: logger.debug( f'Deselected test {item.nodeid} due to "--assignee {assignee}",' f'test has assignee mark: {assignee_marker}') deselected.append(item) continue selected.append(item) # selected will be empty if no filter option was passed, defaulting to full items list items[:] = selected if deselected else items config.hook.pytest_deselected(items=deselected)
def pytest_collection_modifyitems(session, items, config): """Add markers for testimony tokens""" # split the option string and handle no option, single option, multiple # config.getoption(default) doesn't work like you think it does, hence or '' importance = [ i for i in (config.getoption('importance') or '').split(',') if i != '' ] component = [ c for c in (config.getoption('component') or '').split(',') if c != '' ] assignee = [ a for a in (config.getoption('assignee') or '').split(',') if a != '' ] selected = [] deselected = [] logger.info('Processing test items to add testimony token markers') for item in items: if item.nodeid.startswith('tests/robottelo/'): # Unit test, no testimony markers continue # apply the marks for importance, component, and assignee # Find matches from docstrings starting at smallest scope item_docstrings = [ d for d in map(inspect.getdoc, (item.function, getattr(item, 'cls', None), item.module)) if d is not None ] item_mark_names = [m.name for m in item.iter_markers()] for docstring in item_docstrings: # Add marker starting at smallest docstring scope # only add the mark if it hasn't already been applied at a lower scope doc_component = component_regex.findall(docstring) if doc_component and 'component' not in item_mark_names: item.add_marker(pytest.mark.component(doc_component[0])) doc_importance = importance_regex.findall(docstring) if doc_importance and 'importance' not in item_mark_names: item.add_marker(pytest.mark.importance(doc_importance[0])) doc_assignee = assignee_regex.findall(docstring) if doc_assignee and 'assignee' not in item_mark_names: item.add_marker(pytest.mark.assignee(doc_assignee[0])) # exit early if no filters were passed if importance or component or assignee: # Filter test collection based on CLI options for filtering # filters should be applied together # such that --component Repository --importance Critical --assignee jsmith # only collects tests which have all three of these marks # https://github.com/pytest-dev/pytest/issues/1373 Will make this way easier # testimony requires both importance and component, this will blow up if its forgotten importance_marker = item.get_closest_marker('importance').args[0] if importance and importance_marker not in importance: logger.debug( f'Deselected test {item.nodeid} due to "--importance {importance}",' f'test has importance mark: {importance_marker}') deselected.append(item) continue component_marker = item.get_closest_marker('component').args[0] if component and component_marker not in component: logger.debug( f'Deselected test {item.nodeid} due to "--component {component}",' f'test has component mark: {component_marker}') deselected.append(item) continue assignee_marker = item.get_closest_marker('assignee').args[0] if assignee and assignee_marker not in assignee: logger.debug( f'Deselected test {item.nodeid} due to "--assignee {assignee}",' f'test has assignee mark: {assignee_marker}') deselected.append(item) continue selected.append(item) # selected will be empty if no filter option was passed, defaulting to full items list items[:] = selected if deselected else items config.hook.pytest_deselected(items=deselected)