def test_synthesis(request, module_results_df):
# manual test
dct = get_session_synthesis_dct(request,
filter=test_foo,
test_id_format="function")
assert len(dct) == 1
a_param = dct["test_foo[foo]"]["pytest_params"]["a"]
assert not isinstance(a_param, int)
assert str(a_param) == "foo"
dct = get_session_synthesis_dct(request,
filter=test_foo2,
test_id_format="function")
assert len(dct) == 1
name = "test_foo2[foo]" if not pytest2 else "test_foo2[foo[0]-foo[1]]"
i_param = dct[name]["pytest_params"]["i"]
assert not isinstance(i_param, int)
assert str(i_param) == "foo[0]"
j_param = dct[name]["pytest_params"]["j"]
assert not isinstance(j_param, int)
assert str(j_param) == "foo[1]"
# final test on automatic fixture: should be the same
assert module_results_df.dtypes['a'].kind == 'O'
assert module_results_df.dtypes['i'].kind == 'O'
assert module_results_df.dtypes['j'].kind == 'O'
# last note: nothing can be done for test_foo3
# as soon as fixtures are created by @parametrize to handle fixture_ref, you cannot access the values anymore
# TODO can this make it easier ? https://github.com/smarie/python-pytest-harvest/issues/44
assert module_results_df.loc["test_foo3[foo]", "a_value"] == ['r', 1]
assert module_results_df.loc["test_foo3[b]", "a_value"] == (1, 'hello')
def test_alt_usage2_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_alt_usage2,
test_id_format='function')
assert list(results_dct) == [
'test_alt_usage2[b1e+00]', 'test_alt_usage2[b0e+00]'
]
def test_synthesis(request, my_store):
"""
An example test that retrieves synthesis information about this module
"""
# retrieve the synthesis, merged with the fixture store
results_dct = get_session_synthesis_dct(request.session,
filter=test_synthesis.__module__,
durations_in_ms=True,
test_id_format='function',
status_details=False,
fixture_store=my_store,
flatten=True,
flatten_more='my_results_bag')
# print keys and first node details
print("\nKeys:\n" + "\n".join(list(results_dct.keys())))
print("\nFirst node:\n" + "\n".join(
repr(k) + ": " + repr(v)
for k, v in list(results_dct.values())[0].items()))
# convert to a pandas dataframe
results_df = pd.DataFrame.from_dict(results_dct, orient='index')
results_df = results_df.loc[list(results_dct.keys()), :] # fix rows order
results_df.index.name = 'test_id' # set index name
results_df.drop(['pytest_obj'], axis=1,
inplace=True) # drop pytest object column
# print using tabulate
print(tabulate(results_df, headers='keys'))
def test_foo_fixtures_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo,
test_id_format='function')
assert list(results_dct) == [
'test_foo[one_positive_int]',
]
def store(request):
# setup: init the store
store = OrderedDict()
yield store
# teardown: here you can collect all
assert len(store['results_bag']) == 6
print(dict(store['results_bag']))
# retrieve the synthesis, merged with the fixture store
results_dct = get_session_synthesis_dct(request.session,
status_details=False,
durations_in_ms=True,
fixture_store=store,
flatten=True,
flatten_more='results_bag')
# -- use pandas to print
import pandas as pd
results_df = pd.DataFrame.from_dict(results_dct, orient='index')
# (a) remove the full test id path
results_df.index = results_df.index.to_series().apply(
lambda test_id: test_id.split('::')[-1])
# (b) drop pytest object column
results_df.drop(['pytest_obj'], axis=1, inplace=True)
from tabulate import tabulate
print(
tabulate(results_df, headers='keys',
tablefmt="pipe").replace(':-', '--').replace('-:', '--'))
def test_foo_cls_list_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo_cls_list,
test_id_format='function')
ref_list = [
# CasesFoo
'test_foo_cls_list[toto0]',
'test_foo_cls_list[foo0]',
'test_foo_cls_list[hello world0]',
'test_foo_cls_list[two_negative_ints0]',
# strange_ints
'test_foo_cls_list[strange_ints]',
# cases_doc_alternate.py
'test_foo_cls_list[toto1]',
'test_foo_cls_list[foo1]',
'test_foo_cls_list[hello]',
'test_foo_cls_list[two_negative_ints1]',
'test_foo_cls_list[two_negative_ints2]',
# CasesFoo
'test_foo_cls_list[toto2]',
'test_foo_cls_list[foo2]',
'test_foo_cls_list[hello world1]',
'test_foo_cls_list[two_negative_ints3]',
# test_doc_cases.py
'test_foo_cls_list[two_positive_ints]',
'test_foo_cls_list[two_negative_ints4]'
]
if has_pytest_param:
assert list(results_dct) == ref_list
else:
assert len(results_dct) == len(ref_list)
def test_users_synthesis(request, db):
results_dct = get_session_synthesis_dct(request,
filter=test_users,
test_id_format='function')
assert list(results_dct) == [
'test_users[a_is_bob]', 'test_users[a_is_from_db-id=0]',
'test_users[a_is_from_db-id=1]'
]
def test_foo_parametrize_fixture_synthesis(request):
results_dct = get_session_synthesis_dct(
request,
filter=test_foo_parametrize_fixture,
test_id_format='function')
assert list(results_dct) == [
'test_foo_parametrize_fixture[two_positive_ints]',
'test_foo_parametrize_fixture[two_negative_ints]'
]
def test_with_data_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_with_data,
test_id_format='function')
# if has_pytest_param:
assert list(results_dct) == [
'test_with_data[bob-a]', 'test_with_data[bob-b-True]',
'test_with_data[bob-b-False]'
]
def test_foo_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo,
test_id_format='function')
assert list(results_dct) == [
'test_foo[hello]',
'test_foo[simple_generator-who=you]',
# 'test_foo[simple_generator-who=there]' skipped
]
def test_foo_fun_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo_fun,
test_id_format='function')
if has_pytest_param:
assert list(results_dct) == ['test_foo_fun[strange_ints]']
else:
assert list(results_dct) == [
'test_foo_fun[strange_ints[0]-strange_ints[1]]'
]
def test_alt_usage1_synthesis(request):
results_dct = get_session_synthesis_dct(request, filter=test_alt_usage1, test_id_format='function')
if sys.version_info > (3, 6):
assert list(results_dct) == [
'test_alt_usage1[a=True,b=1.2 and -1]',
'test_alt_usage1[a=True,b=1.2 and 2]',
'test_alt_usage1[a=True,b=0.0 and -1]',
'test_alt_usage1[a=True,b=0.0 and 2]'
]
else:
assert len(results_dct) == 4
def test_synthesis2(request, fixture_store):
results_dct2 = get_session_synthesis_dct(request,
filter=test_fixture_ref2,
test_id_format='function',
fixture_store=fixture_store,
flatten=True)
assert list(results_dct2) == [
'test_fixture_ref2[a-1]', 'test_fixture_ref2[2-b-5]',
'test_fixture_ref2[2-b-6]', 'test_fixture_ref2[c0]',
'test_fixture_ref2[c1]'
]
def test_synthesis_id_formatting(request):
"""
Note: we could do this at many other places (hook, teardown of a session-scope fixture...)
Note2: we could provide helper methods in pytest_harvest to perform the code below more easily
:param request:
:param store:
:return:
"""
# Get session synthesis filtered on the test function of interest
# -- to debug the filter:
# assert pytest_item_matches_filter(request.session.items[28], filter={TestX.test_easy})
fmt = 'function'
results_dct = get_session_synthesis_dct(request.session,
filter=TestX.test_easy,
test_id_format=fmt)
assert list(results_dct.keys())[0] == 'test_easy[True]'
fmt = 'class'
results_dct = get_session_synthesis_dct(request.session,
filter=TestX.test_easy,
test_id_format=fmt)
assert list(results_dct.keys())[0] == 'TestX::()::test_easy[True]'
fmt = 'module'
results_dct = get_session_synthesis_dct(request.session,
filter=TestX.test_easy,
test_id_format=fmt)
# this does not work when we run the test from the meta-tester
# assert list(results_dct.keys())[0] == 'test_get_session_results.py::TestX::()::test_easy[True]'
pattern_str = re.escape("test_get_session_results.py::TestX::()::test_easy[True]") \
.replace(re.escape('test_get_session_results'), '^[a-zA-Z0-9_]*?') # replace the file name with a non-greedy capturer
assert re.match(pattern_str, list(results_dct.keys())[0])
def fmt(test_id):
return test_id.split('::')[-1].lower()
results_dct = get_session_synthesis_dct(request.session,
filter=TestX.test_easy,
test_id_format=fmt)
assert list(results_dct.keys())[0] == 'test_easy[true]'
def test_foo_default_cases_file_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo_default_cases_file,
test_id_format='function')
assert list(results_dct) == [
'test_foo_default_cases_file[%s]' %
('two_positive_ints'
if has_pytest_param else 'two_positive_ints[0]-two_positive_ints[1]'),
'test_foo_default_cases_file[%s]' %
('two_negative_ints'
if has_pytest_param else 'two_negative_ints[0]-two_negative_ints[1]')
]
def test_foo_cls_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo_cls,
test_id_format='function')
if has_pytest_param:
assert list(results_dct) == [
'test_foo_cls[hello world]', 'test_foo_cls[two_negative_ints]'
]
else:
assert list(results_dct) == [
'test_foo_cls[hello world[0]-hello world[1]]',
'test_foo_cls[two_negative_ints[0]-two_negative_ints[1]]'
]
def test_synthesis1(request, fixture_store):
results_dct1 = get_session_synthesis_dct(request,
filter=test_fixture_ref1,
test_id_format='function',
fixture_store=fixture_store,
flatten=True)
assert [(k, v['test_fixture_ref1_arg'])
for k, v in results_dct1.items()] == [
('test_fixture_ref1[arg_is_c]', 'c'),
('test_fixture_ref1[arg_is_a]', 'a'),
('test_fixture_ref1[arg_is_b-5]', 'b5'),
('test_fixture_ref1[arg_is_b-6]', 'b6'),
]
def test_synthesis_contains_everything(request):
""" Tests that the synthesis contains all test nodes """
# retrieve session results
synth_dct = get_session_synthesis_dct(request, filter_incomplete=False)
# ref list is the list of all test items defined in this file.
these_tests = [item.nodeid for item in request.session.items if this_file_name in item.nodeid]
print(these_tests)
assert len(these_tests) == 19
# check that synth_dct contains all these test nodes
missing = set(these_tests) - set(synth_dct.keys())
assert len(missing) == 0
def test_idgen1_synthesis(request):
results_dct = get_session_synthesis_dct(request, filter=test_idgen1, test_id_format='function')
if sys.version_info >= (3, 6):
if PYTEST3_OR_GREATER:
assert list(results_dct) == [
'test_idgen1[10yes-c2.1-a=True,b= -1]',
'test_idgen1[10yes-c2.1-a=False,b= 3]',
'test_idgen1[10yes-c0.0-a=True,b= -1]',
'test_idgen1[10yes-c0.0-a=False,b= 3]'
]
else:
# the order seems not guaranteed or at least quite different in pytest 2
assert len(results_dct) == 4
else:
assert len(results_dct) == 4
def test_foo_alternate_cases_file_and_two_marked_skip_synthesis(request):
results_dct = get_session_synthesis_dct(
request,
filter=test_foo_alternate_cases_file_and_two_marked_skip,
test_id_format='function')
if has_pytest_param:
assert list(results_dct) == [
'test_foo_alternate_cases_file_and_two_marked_skip[hello]',
'test_foo_alternate_cases_file_and_two_marked_skip[two_negative_ints0]',
'test_foo_alternate_cases_file_and_two_marked_skip[two_negative_ints1]'
]
else:
assert list(results_dct) == [
'test_foo_alternate_cases_file_and_two_marked_skip[0hello[0]-hello[1]]',
'test_foo_alternate_cases_file_and_two_marked_skip[2two_negative_ints[0]-two_negative_ints[1]]',
'test_foo_alternate_cases_file_and_two_marked_skip[4two_negative_ints[0]-two_negative_ints[1]]'
]
def test_synthesis(request):
"""
Tests that this test runs in the right order (second)
This tests that we use correctly the pytest hack "fun.place_as = func"
See https://github.com/pytest-dev/pytest/issues/4429
"""
# Get session synthesis
# - filtered on the test function of interest
# - combined with our store
results_dct = get_session_synthesis_dct(request.session, filter=test_synthesis.__module__,
durations_in_ms=True, test_id_format='function')
# incomplete are not here so length should be 1
assert len(results_dct) == 2
it = list(results_dct.values())
assert it[0]['pytest_obj'] == test_basic_b
assert it[1]['pytest_obj'] == test_basic_gen_b
def pytest_sessionfinish(session, exitstatus):
"""Handle the end of the session."""
n = session.config.option.durations
if n is None:
return
print('\n')
try:
import pytest_harvest
except ImportError:
print('Module-level timings require pytest-harvest')
return
from py.io import TerminalWriter
# get the number to print
res = pytest_harvest.get_session_synthesis_dct(session)
files = dict()
for key, val in res.items():
parts = Path(key.split(':')[0]).parts
# split mne/tests/test_whatever.py into separate categories since these
# are essentially submodule-level tests. Keeping just [:3] works,
# except for mne/viz where we want level-4 granulatity
split_submodules = (('mne', 'viz'), ('mne', 'preprocessing'))
parts = parts[:4 if parts[:2] in split_submodules else 3]
if not parts[-1].endswith('.py'):
parts = parts + ('', )
file_key = '/'.join(parts)
files[file_key] = files.get(file_key, 0) + val['pytest_duration_s']
files = sorted(list(files.items()), key=lambda x: x[1])[::-1]
# print
files = files[:n]
if len(files):
writer = TerminalWriter()
writer.line() # newline
writer.sep('=', f'slowest {n} test module{_pl(n)}')
names, timings = zip(*files)
timings = [f'{timing:0.2f}s total' for timing in timings]
rjust = max(len(timing) for timing in timings)
timings = [timing.rjust(rjust) for timing in timings]
for name, timing in zip(names, timings):
writer.line(f'{timing.ljust(15)}{name}')
def test_foo_multi_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo_multi,
test_id_format='function')
if sys.version_info >= (3, 6):
if has_pytest_param:
assert list(results_dct) == [
'test_foo_multi[hello]',
# 'test_foo_multi[simple_generator-who=you]', skipped
# 'test_foo_multi[simple_generator-who=you]', skipped
'test_foo_multi[simple_generator-who=there-a=5-b=5]',
'test_foo_multi[simple_generator-who=there-a=10-b=10]'
]
else:
assert list(results_dct) == [
'test_foo_multi[hello[0]-hello[1]]',
# 'test_foo_multi[simple_generator-who=you]', skipped
# 'test_foo_multi[simple_generator-who=you]', skipped
'test_foo_multi[simple_generator-who=there-a=5-b=5[0]-simple_generator-who=there-a=5-b=5[1]]',
'test_foo_multi[simple_generator-who=there-a=10-b=10[0]-simple_generator-who=there-a=10-b=10[1]]'
]
else:
assert len(results_dct) == 3
def test_synthesis_failed(request):
""" Tests that the synthesis concerning the failed test is correct """
synth_dct = get_session_synthesis_dct(request.session, filter=[test_failing])
for test_id, v in synth_dct.items():
assert v['pytest_status'] == 'failed'
def test_foo_synthesis_all_options(request, flatten, durations_in_ms):
"""
Tests that the synthesis is ok
:param request:
:return:
"""
# Get the synthesis dictionary concerning `test_foo`
synth_dct = get_session_synthesis_dct(request.session, status_details=True, flatten=flatten, filter=test_foo,
durations_in_ms=durations_in_ms)
# from pprint import pprint
# pprint(dict(synth_dct))
durations_unit = ('ms' if durations_in_ms else 's')
# Check the returned dictionary contents
prefix = '' if flatten else 'pytest_'
expected_keys = {'pytest_obj',
prefix + 'status',
prefix + 'duration_' + durations_unit}
stages = ['setup', 'call', 'teardown']
if not flatten:
expected_keys.update({prefix + 'status_details', prefix + 'params'})
else:
expected_keys.update({(prefix + 'status__' + stage) for stage in stages})
# add parameters
expected_keys.update({mark.args[0] for mark in get_pytest_parametrize_marks(test_foo)})
# add parametrized fixtures
expected_keys.update({parametrized_fixture.__name__ + '_param' for parametrized_fixture in [a_number_str]})
# compute the parameter values for all tests in order
params = list(product(fixture_params, test_params))
for i, (nodeid, nodeinfo) in enumerate(synth_dct.items()):
# check that all keys are present
assert set(nodeinfo.keys()) == expected_keys
# main test information
assert nodeinfo['pytest_obj'] == test_foo # check that the filter worked
assert nodeinfo[prefix + 'status'] == 'passed'
assert nodeinfo[prefix + 'duration_' + durations_unit] >= 0
# test status details
if not flatten:
assert set(nodeinfo[prefix + 'status_details'].keys()) == set(stages)
for step in stages:
if flatten:
step_info = nodeinfo[prefix + 'status__' + step]
else:
step_info = nodeinfo[prefix + 'status_details'][step]
assert len(step_info) == 2
assert step_info[0] == 'passed'
assert step_info[1] >= 0
# parameter values
if flatten:
param_dct = nodeinfo
else:
assert set(nodeinfo[prefix + 'params'].keys()) == {'p', 'a_number_str_param'}
param_dct = nodeinfo[prefix + 'params']
assert param_dct['a_number_str_param'] == params[i][0]
assert param_dct['p'] == params[i][1]
def test_foo1_synthesis(request):
results_dct = get_session_synthesis_dct(request,
filter=test_foo1,
test_id_format='function')
assert list(results_dct) == ['test_foo1[1-2]', 'test_foo1[-1--2]']
def test_synthesis(flatten, flatten_more, request, store):
"""Tests that the synthesis dictionary combined with with fixture store is ok"""
# retrieve the synthesis, merged with the fixture store
results_dct = get_session_synthesis_dct(request.session,
filter=test_complete,
status_details=False,
fixture_store=store,
flatten=flatten,
flatten_more=flatten_more)
# ------ PRINTS ---------
# --test node ids
print("\n".join(list(results_dct.keys())))
# --zoom on first node
print("\n".join(
repr(k) + ": " + repr(v)
for k, v in list(results_dct.values())[0].items()))
if flatten and flatten_more:
# -- use tabulate to print
from tabulate import tabulate
# print(tabulate(dct, headers='keys')) not possible, it does not yet support that dict keys represent rows
# -- use pandas to print
import pandas as pd
results_df = pd.DataFrame.from_dict(results_dct, orient='index')
# (a) remove the full test id path
results_df.index = results_df.index.to_series().apply(
lambda test_id: test_id.split('::')[-1])
# (b) replace pytest object with its name
results_df['pytest_obj'] = results_df['pytest_obj'].map(
lambda f: f.__name__)
print(
tabulate(results_df, headers='keys',
tablefmt="pipe").replace(':-', '--').replace('-:', '--'))
# ------ ASSERTS ---------
# compute the parameter values for all tests in order
params = list(product(fixture_params, test_params))
assert len(results_dct) == len(params)
for i, (nodeid, node_info) in enumerate(results_dct.items()):
if flatten:
where_dct = node_info
if flatten_more is None:
where_dct_results = node_info['my_results']
else:
where_dct_results = node_info
else:
where_dct = node_info['fixtures']
where_dct_results = node_info['fixtures']['my_results']
assert where_dct['my_fix'] == 'my_fix #' + params[i][0]
assert where_dct_results['score'] == params[i][1] * 10
assert where_dct_results['what'] == 'hello my_fix #' + params[i][0]
