def test_coverage(testdir):
testdir.makepyfile(
# Setup file to cover:
lib="""
def mysum(a, b):
return a + b
def myprod(a, b):
return a * b
""",
# Setup python file to cover mysum function
test_lib="""
import lib
def test_sum():
assert lib.mysum(1, 3) == 4
assert lib.mysum("cat", "dog") == "catdog"
assert lib.mysum(1.5, 2) == 3.5
""",
)
# Setup notebook to cover myprod function
nb = build_nb([
"import lib",
"lib.myprod(1, 3)",
"lib.myprod(2.5, 2.5)",
"lib.myprod(2, 'cat')"
], mark_run=True)
add_expected_plaintext_outputs(nb, [
None, "3", "6.25", "'catcat'"
])
# Write notebook to test dir
nbformat.write(nb, os.path.join(
str(testdir.tmpdir), 'test_coverage.ipynb'))
# Run tests
result = testdir.runpytest_inprocess('--nbval', '--current-env', '--cov', '.')
# Check tests went off as they should:
assert result.ret == 0
# Ensure coverage report was generated:
assert os.path.exists(os.path.join(str(testdir.tmpdir), '.coverage'))
# Check that all lines were covered:
result.stdout.fnmatch_lines([
'lib.py*4*0*100%'
])
def test_conf_ignore_stderr(testdir):
# Setup test config
testdir.makeconftest(_ignore_stderr_code)
# Setup notebook with stream outputs
nb = build_nb([
"import sys",
"sys.stdout.write('test\\n')",
"sys.stderr.write('error output\\n')",
"sys.stdout.write('test\\n')\nsys.stderr.write('error output\\n')",
],
mark_run=True)
nb.cells[1].outputs.append(
nbformat.v4.new_output(
'stream',
text=u'test\n',
))
nb.cells[2].outputs.append(
nbformat.v4.new_output(
'stream',
name='stderr',
text=u'different error output',
))
nb.cells[3].outputs.append(
nbformat.v4.new_output(
'stream',
text=u'test\n',
))
nb.cells[3].outputs.append(
nbformat.v4.new_output(
'stream',
name='stderr',
text=u'different error output',
))
# Write notebook to test dir
nbformat.write(nb, os.path.join(str(testdir.tmpdir), 'test_ignore.ipynb'))
# Run tests
result = testdir.runpytest_subprocess('--nbval', '--nbval-current-env',
'.')
# Check tests went off as they should:
assert result.ret == 0
def test_coverage(testdir):
testdir.makepyfile(
# Setup file to cover:
lib="""
def mysum(a, b):
return a + b
def myprod(a, b):
return a * b
""",
# Setup python file to cover mysum function
test_lib="""
import lib
def test_sum():
assert lib.mysum(1, 3) == 4
assert lib.mysum("cat", "dog") == "catdog"
assert lib.mysum(1.5, 2) == 3.5
""",
)
# Setup notebook to cover myprod function
nb = build_nb([
"import lib", "lib.myprod(1, 3)", "lib.myprod(2.5, 2.5)",
"lib.myprod(2, 'cat')"
],
mark_run=True)
add_expected_plaintext_outputs(nb, [None, "3", "6.25", "'catcat'"])
# Write notebook to test dir
nbformat.write(nb, os.path.join(str(testdir.tmpdir),
'test_coverage.ipynb'))
# Run tests
result = testdir.runpytest_inprocess('--nbval', '--current-env', '--cov',
'.')
# Check tests went off as they should:
assert result.ret == 0
# Ensure coverage report was generated:
assert os.path.exists(os.path.join(str(testdir.tmpdir), '.coverage'))
# Check that all lines were covered:
result.stdout.fnmatch_lines(['lib.py*4*0*100%'])
def test_conf_ignore_stderr(testdir):
# Setup test config
testdir.makeconftest(_ignore_stderr_code)
# Setup notebook with stream outputs
nb = build_nb([
"import sys",
"sys.stdout.write('test\\n')",
"sys.stderr.write('error output\\n')",
"sys.stdout.write('test\\n')\nsys.stderr.write('error output\\n')",
], mark_run=True)
nb.cells[1].outputs.append(nbformat.v4.new_output(
'stream',
text=u'test\n',
))
nb.cells[2].outputs.append(nbformat.v4.new_output(
'stream',
name='stderr',
text=u'different error output',
))
nb.cells[3].outputs.append(nbformat.v4.new_output(
'stream',
text=u'test\n',
))
nb.cells[3].outputs.append(nbformat.v4.new_output(
'stream',
name='stderr',
text=u'different error output',
))
# Write notebook to test dir
nbformat.write(nb, os.path.join(
str(testdir.tmpdir), 'test_ignore.ipynb'))
# Run tests
result = testdir.runpytest_subprocess('--nbval', '--current-env', '.')
# Check tests went off as they should:
assert result.ret == 0
def test_nbdime(testdir, mocker):
# Make a test notebook where output doesn't match input
nb = build_nb(["1+1", "1+1"], mark_run=True)
add_expected_plaintext_outputs(nb, ["2", "3"])
# Write notebook to test dir
filename = 'test_nbdime.ipynb'
nbformat.write(nb, os.path.join(str(testdir.tmpdir), filename))
# patch the run_server function so that it doesn't actually
# spawn a server and display the diff. But the diff is still
# calculated.
mocker.patch('nbdime.webapp.nbdiffweb.run_server')
result = testdir.runpytest_inprocess('--nbval', '--nbval-current-env',
'--nbdime', filename)
# run_server() is only called if there is a discrepancy in the notebook.
# so it should have been called in this case:
nbdime.webapp.nbdiffweb.run_server.assert_called_once()
# note: this import must be AFTER the mocker.patch
from nbval.nbdime_reporter import EXIT_TESTSFAILED
assert result.ret == EXIT_TESTSFAILED
def test_timeouts(testdir):
# This test uses the testdir fixture from pytester, which is useful for
# testing pytest plugins. It writes a notebook to a temporary dir
# and then runs pytest.
# Note: The test and the notebook are defined together in order to
# emphasize the close dependence of the test on the structure and
# content of the notebook
# Setup notebook to test:
sources = [
# In [1]:
"from time import sleep",
# In [2]:
"for i in range(100000):\n" + " sleep(1)\n" + "myvar = 5",
# In [3]:
"a = 5",
# In [4]:
"print(myvar)",
# In [5]:
"import signal\n" + "signal.signal(signal.SIGINT, signal.SIG_IGN)\n" +
"for i in range(1000):\n" + " sleep(100)",
# In [6]:
"b = 5",
]
nb = build_nb(sources)
# Write notebook to test dir
nbformat.write(nb, os.path.join(str(testdir.tmpdir),
'test_timeouts.ipynb'))
# Run tests
result = testdir.inline_run('--nbval', '--nbval-current-env',
'--nbval-cell-timeout', '5', '-s')
reports = result.getreports('pytest_runtest_logreport')
# Setup and teardown of cells should have no issues:
setup_teardown = [r for r in reports if r.when != 'call']
for r in setup_teardown:
assert r.passed
reports = [r for r in reports if r.when == 'call']
assert len(reports) == 6
# Import cell should pass:
if not reports[0].passed:
pytest.fail("Inner exception: %s" % reports[0].longreprtext)
# First timeout cell should fail, unexpectedly
assert reports[1].failed and not hasattr(reports[1], 'wasxfail')
# Normal cell after timeout should pass, but be expected to fail
if not reports[2].passed:
pytest.fail("Inner exception: %s" % reports[2].longreprtext)
assert hasattr(reports[2], 'wasxfail')
# Cell trying to access variable declare after loop in timeout
# should fail, expectedly (marked skipped)
assert reports[3].skipped and hasattr(reports[3], 'wasxfail')
# Second timeout loop should fail, expectedly, and cause all following to fail
assert reports[4].skipped and hasattr(reports[4], 'wasxfail')
assert reports[5].skipped and hasattr(reports[5], 'wasxfail')
