def pytest_collection_modifyitems(session, config, items):
"""Remove/Include stubbed tests from collection based on CLI option"""
opt_passed = config.getvalue('mark_manuals_passed')
opt_skipped = config.getvalue('mark_manuals_skipped')
# TODO turn this into a flag or a choice option, this logic is just silly.
mark_skipped = opt_skipped and not opt_passed
include_stubbed = config.getvalue('include_stubbed')
selected = []
deselected = []
for item in items:
stub_marked = item.get_closest_marker(name='stubbed')
# The test case is stubbed, and --include-stubbed was passed, include in collection
if stub_marked and include_stubbed:
selected.append(item)
# enforce skip/pass behavior by marking skip
if mark_skipped:
logger.debug(
f'Marking collected stubbed test "{item.nodeid}" to skip')
item.add_marker(marker=pytest.mark.skip(
reason=NOT_IMPLEMENTED))
continue
# The test case is stubbed, but --include-stubbed was NOT passed, deselect the item
if stub_marked and not include_stubbed:
logger.debug(f'Deselecting stubbed test {item.nodeid}, '
'use --include-stubbed to include in collection')
deselected.append(item)
continue
# Its a non-stubbed item, this hook doesn't apply
selected.append(item)
config.hook.pytest_deselected(items=deselected)
items[:] = selected
def pytest_collection_modifyitems(items, config):
"""
Collects and modifies tests collection based on pytest option to deselect tests for new infra
"""
include_onprem_provision = config.getoption('include_onprem_provisioning',
False)
include_libvirt = config.getoption('include_libvirt', False)
include_eauth = config.getoption('include_external_auth', False)
include_vlan = config.getoption('include_vlan_networking', False)
include_non_satci_tests = config.getvalue('include_non_satci_tests').split(
',')
selected = []
deselected = []
# Cloud Provisioning Test can be run on new pipeline
for item in items:
# Include/Exclude tests those are not part of SatQE CI
item_component = item.get_closest_marker('component')
if item_component and (item_component.args[0] in non_satCI_components):
if item_component.args[0] in include_non_satci_tests:
selected.append(item)
else:
deselected.append(item)
continue
item_marks = [m.name for m in item.iter_markers()]
# Include / Exclude On Premises Provisioning Tests
if 'on_premises_provisioning' in item_marks:
selected.append(
item) if include_onprem_provision else deselected.append(item)
continue
# Include / Exclude External Libvirt based Tests
if 'libvirt_discovery' in item_marks:
selected.append(item) if include_libvirt else deselected.append(
item)
continue
# Include / Exclude External Auth based Tests
if 'external_auth' in item_marks:
selected.append(item) if include_eauth else deselected.append(item)
continue
# Include / Exclude VLAN networking based based Tests
if 'vlan_networking' in item_marks:
selected.append(item) if include_vlan else deselected.append(item)
continue
# This Plugin does not applies to this test
selected.append(item)
logger.debug(
f'Selected {len(selected)} and deselected {len(deselected)} '
'tests based on auto un-collectable markers and pytest options.')
items[:] = selected or items
config.hook.pytest_deselected(items=deselected)
def pytest_collection_modifyitems(session, items, config):
"""Generate the issue collection (using bz cache via bugzilla issue handler util)
This collection includes pre-processed `is_open` status for each issue
"""
# generate_issue_collection will save a file, set by --bz-cache value
pytest.issue_data = generate_issue_collection(items, config)
# Add skipif markers, and modify collection based on --bz option
bz_filters = config.getoption('BZ', None)
if bz_filters:
bz_filters = bz_filters.split(',')
selected = []
deselected = []
for item in items:
# Add a skipif marker for the issues
skip_if_open = item.get_closest_marker('skip_if_open')
if skip_if_open:
# marker must have `BZ:123456` as argument.
issue = skip_if_open.kwargs.get('reason') or skip_if_open.args[0]
item.add_marker(pytest.mark.skipif(is_open(issue), reason=issue))
# remove items from collection
if bz_filters:
# Only include items which have BZ mark that includes any of the filtered bz numbers
item_bz_marks = set(getattr(item.get_closest_marker('BZ', None), 'args', []))
if bool(set(bz_filters) & item_bz_marks):
selected.append(item)
else:
logger.debug(
f'Deselected test [{item.nodeid}] due to BZ filter {bz_filters} '
f'and available marks {item_bz_marks}'
)
deselected.append(item)
continue # move to the next item
# append only when there is a BZ in common between the lists
else:
selected.append(item)
config.hook.pytest_deselected(items=deselected)
items[:] = selected
def pytest_collection_modifyitems(items, config):
"""
Collects and modifies tests collection based on pytest option to deselect tests for new infra
"""
include_onprem_provision = config.getoption('include_onprem_provisioning', False)
include_libvirt = config.getoption('include_libvirt', False)
include_eauth = config.getoption('include_external_auth', False)
include_vlan = config.getoption('include_vlan_networking', False)
selected = []
deselected = []
# Cloud Provisioning Test can be run on new pipeline
for item in items:
item_marks = [m.name for m in item.iter_markers()]
# Include / Exclude On Premises Provisioning Tests
if 'on_premises_provisioning' in item_marks:
selected.append(item) if include_onprem_provision else deselected.append(item)
continue
# Include / Exclude External Libvirt based Tests
if any(marker in item_marks for marker in ['libvirt_discovery', 'libvirt_content_host']):
selected.append(item) if include_libvirt else deselected.append(item)
continue
# Include / Exclude External Auth based Tests
if 'external_auth' in item_marks:
selected.append(item) if include_eauth else deselected.append(item)
continue
# Include / Exclude VLAN networking based based Tests
if 'vlan_networking' in item_marks:
selected.append(item) if include_vlan else deselected.append(item)
continue
# This Plugin does not applies to this test
selected.append(item)
logger.debug(
f'Selected {len(selected)} and deselected {len(deselected)} '
'tests based on new infra markers.'
)
items[:] = selected or items
config.hook.pytest_deselected(items=deselected)
def pytest_collection_modifyitems(session, items, config):
"""Called after collection has been performed, may filter or re-order
the items in-place.
"""
collection_logger.debug(f'Collected {len(items)} test cases')
# Modify items based on collected issue_data
deselected_items = []
for item in items:
# 1. Deselect tests marked with @pytest.mark.deselect
# WONTFIX BZs makes test to be dynamically marked as deselect.
deselect = item.get_closest_marker('deselect')
if deselect:
deselected_items.append(item)
reason = deselect.kwargs.get('reason', deselect.args)
collection_logger.debug(
f'Deselected test "{item.name}" reason: {reason}')
# Do nothing more with deselected tests
continue
config.hook.pytest_deselected(items=deselected_items)
items[:] = [item for item in items if item not in deselected_items]
def pytest_collection_modifyitems(session, items, config):
"""Add markers and user_properties for testimony token metadata
user_properties is used by the junit plugin, and thus by many test report systems
Handle test function/class/module/session scope metadata coming from test docblocks
Apply user_properties, ibutsu metadata, and pytest markers
Markers for metadata use the testimony token name as the mark name
The value of the token for the mark is the first mark arg
Control test collection for custom options related to testimony metadata
"""
# get RHEL version of the satellite
rhel_version = get_sat_rhel_version().base_version
sat_version = get_sat_version().base_version
snap_version = settings.server.version.get('snap', '')
# split the option string and handle no option, single option, multiple
# config.getoption(default) doesn't work like you think it does, hence or ''
importance = [
i for i in (config.getoption('importance') or '').split(',') if i != ''
]
component = [
c for c in (config.getoption('component') or '').split(',') if c != ''
]
assignee = [
a for a in (config.getoption('assignee') or '').split(',') if a != ''
]
selected = []
deselected = []
logger.info('Processing test items to add testimony token markers')
for item in items:
if item.nodeid.startswith(
'tests/robottelo/') and 'test_junit' not in item.nodeid:
# Unit test, no testimony markers
continue
# apply the marks for importance, component, and assignee
# Find matches from docstrings starting at smallest scope
item_docstrings = [
d for d in map(inspect.getdoc, (item.function,
getattr(item, 'cls', None),
item.module)) if d is not None
]
item_mark_names = [m.name for m in item.iter_markers()]
for docstring in item_docstrings:
# Add marker starting at smallest docstring scope
# only add the mark if it hasn't already been applied at a lower scope
doc_component = component_regex.findall(docstring)
if doc_component and 'component' not in item_mark_names:
item.add_marker(pytest.mark.component(doc_component[0]))
doc_importance = importance_regex.findall(docstring)
if doc_importance and 'importance' not in item_mark_names:
item.add_marker(pytest.mark.importance(doc_importance[0]))
doc_assignee = assignee_regex.findall(docstring)
if doc_assignee and 'assignee' not in item_mark_names:
item.add_marker(pytest.mark.assignee(doc_assignee[0]))
# add markers as user_properties so they are recorded in XML properties of the report
for marker in item.iter_markers():
item.user_properties.append(
(marker.name, next(iter(marker.args), None)))
item.user_properties.append(("BaseOS", rhel_version))
item.user_properties.append(("SatelliteVersion", sat_version))
item.user_properties.append(("SnapVersion", snap_version))
item.user_properties.append(
("start_time",
datetime.datetime.utcnow().strftime(FMT_XUNIT_TIME)))
# add custom ibutsu metadata fields for test case grouping and heatmaps
if hasattr(item, "_ibutsu"):
item._ibutsu["data"]["metadata"].update({
# TODO Work with ibutsu team, better mechanism for defining 'special' data
# TODO Add sat version to this item data at test execution time
"component":
item.get_closest_marker('component').args[0]
})
# exit early if no filters were passed
if importance or component or assignee:
# Filter test collection based on CLI options for filtering
# filters should be applied together
# such that --component Repository --importance Critical --assignee jsmith
# only collects tests which have all three of these marks
# https://github.com/pytest-dev/pytest/issues/1373 Will make this way easier
# testimony requires both importance and component, this will blow up if its forgotten
importance_marker = item.get_closest_marker('importance').args[0]
if importance and importance_marker not in importance:
logger.debug(
f'Deselected test {item.nodeid} due to "--importance {importance}",'
f'test has importance mark: {importance_marker}')
deselected.append(item)
continue
component_marker = item.get_closest_marker('component').args[0]
if component and component_marker not in component:
logger.debug(
f'Deselected test {item.nodeid} due to "--component {component}",'
f'test has component mark: {component_marker}')
deselected.append(item)
continue
assignee_marker = item.get_closest_marker('assignee').args[0]
if assignee and assignee_marker not in assignee:
logger.debug(
f'Deselected test {item.nodeid} due to "--assignee {assignee}",'
f'test has assignee mark: {assignee_marker}')
deselected.append(item)
continue
selected.append(item)
# selected will be empty if no filter option was passed, defaulting to full items list
items[:] = selected if deselected else items
config.hook.pytest_deselected(items=deselected)
def pytest_collection_modifyitems(session, items, config):
"""Add markers for testimony tokens"""
# split the option string and handle no option, single option, multiple
# config.getoption(default) doesn't work like you think it does, hence or ''
importance = [
i for i in (config.getoption('importance') or '').split(',') if i != ''
]
component = [
c for c in (config.getoption('component') or '').split(',') if c != ''
]
assignee = [
a for a in (config.getoption('assignee') or '').split(',') if a != ''
]
selected = []
deselected = []
logger.info('Processing test items to add testimony token markers')
for item in items:
if item.nodeid.startswith('tests/robottelo/'):
# Unit test, no testimony markers
continue
# apply the marks for importance, component, and assignee
# Find matches from docstrings starting at smallest scope
item_docstrings = [
d for d in map(inspect.getdoc, (item.function,
getattr(item, 'cls', None),
item.module)) if d is not None
]
item_mark_names = [m.name for m in item.iter_markers()]
for docstring in item_docstrings:
# Add marker starting at smallest docstring scope
# only add the mark if it hasn't already been applied at a lower scope
doc_component = component_regex.findall(docstring)
if doc_component and 'component' not in item_mark_names:
item.add_marker(pytest.mark.component(doc_component[0]))
doc_importance = importance_regex.findall(docstring)
if doc_importance and 'importance' not in item_mark_names:
item.add_marker(pytest.mark.importance(doc_importance[0]))
doc_assignee = assignee_regex.findall(docstring)
if doc_assignee and 'assignee' not in item_mark_names:
item.add_marker(pytest.mark.assignee(doc_assignee[0]))
# exit early if no filters were passed
if importance or component or assignee:
# Filter test collection based on CLI options for filtering
# filters should be applied together
# such that --component Repository --importance Critical --assignee jsmith
# only collects tests which have all three of these marks
# https://github.com/pytest-dev/pytest/issues/1373 Will make this way easier
# testimony requires both importance and component, this will blow up if its forgotten
importance_marker = item.get_closest_marker('importance').args[0]
if importance and importance_marker not in importance:
logger.debug(
f'Deselected test {item.nodeid} due to "--importance {importance}",'
f'test has importance mark: {importance_marker}')
deselected.append(item)
continue
component_marker = item.get_closest_marker('component').args[0]
if component and component_marker not in component:
logger.debug(
f'Deselected test {item.nodeid} due to "--component {component}",'
f'test has component mark: {component_marker}')
deselected.append(item)
continue
assignee_marker = item.get_closest_marker('assignee').args[0]
if assignee and assignee_marker not in assignee:
logger.debug(
f'Deselected test {item.nodeid} due to "--assignee {assignee}",'
f'test has assignee mark: {assignee_marker}')
deselected.append(item)
continue
selected.append(item)
# selected will be empty if no filter option was passed, defaulting to full items list
items[:] = selected if deselected else items
config.hook.pytest_deselected(items=deselected)