"""
Test notebooks in doc/
"""
import subprocess
from pathlib import Path
from glob import glob
# we have to use this, nbconvert removes cells that execute shell comands
import jupytext
import pytest
from conftest import _path_to_tests
path_to_doc = _path_to_tests().parent / 'doc'
nbs = [
f for f in glob(str(Path(path_to_doc, '**', '*.ipynb')))
if 'auto_examples' not in f
]
# we cannot use papermill since some notebooks use papermill via NotebookRunner
# there is an issue when this happens, so we just run it as scripts using
# ipython directly
def run_notebook(nb):
print('Running %s' % nb)
out = 'nb.py'
jupytext.write(jupytext.read(nb), out)
# jupytext keeps shell commands but adds them as comments, fix
"""
Runs examples from examples/
"""
import os
import pytest
import subprocess
from conftest import _path_to_tests
_examples = [
f for f in os.listdir(str(_path_to_tests().parent / 'examples'))
if f.endswith('.py')
]
def test_pipeline_runs(tmp_intermediate_example_directory):
assert subprocess.call(['python', 'pipeline.py']) == 0
# def test_partitioned_execution_runs(tmp_example_pipeline_directory):
# assert subprocess.call(['python', 'partitioned_execution.py']) == 0
@pytest.mark.parametrize('name', _examples)
def test_examples(tmp_examples_directory, name):
# TODO: add timeout
assert subprocess.call(['python', name]) == 0
"""
Runs examples form examples/
"""
import os
import pytest
import subprocess
from conftest import _path_to_tests
_recipes = [
f for f in os.listdir(str(_path_to_tests().parent / 'recipes'))
if f.endswith('.py')
]
@pytest.mark.parametrize('name', _recipes)
def test_examples(tmp_recipes_directory, name):
# TODO: add timeout
assert subprocess.call(['python', name]) == 0
def create_engine_with_schema(schema):
def fake_create_engine(*args, **kwargs):
if 'sqlite' in args[0]:
return create_engine(*args, **kwargs)
else:
return create_engine(
*args,
**kwargs,
connect_args=dict(options=f'-c search_path={schema}'))
return fake_create_engine
@fixture_tmp_dir(_path_to_tests() / 'assets' / 'pipeline-sql')
def tmp_pipeline_sql():
pass
@fixture_tmp_dir(_path_to_tests() / 'assets' / 'pipeline-r')
def tmp_pipeline_r():
pass
@fixture_tmp_dir(_path_to_tests() / 'assets' /
'pipeline-sql-products-in-source')
def tmp_pipeline_sql_products_in_source():
pass
Runs notebooks under doc/
NOTE: we cannot run them using papermill.execute_notebook as some notebooks
might run ploomber.NotebookRunner tasks inside them, which will cause
a nested call to papermill.execute_notebook.
"""
import subprocess
from pathlib import Path
from glob import glob
import pytest
from conftest import _path_to_tests
import jupytext
import nbformat
PATH_TO_DOC = str(_path_to_tests().parent / 'doc')
# ignore auto-generated notebooks
NBS = [
nb for nb in glob(str(Path(PATH_TO_DOC, '**/*.ipynb')))
if 'auto_examples' not in nb
]
@pytest.mark.parametrize('path', NBS)
def test_notebooks(tmp_directory, path):
path = Path(path)
nb = nbformat.reads(path.read_text(), as_version=nbformat.NO_CONVERT)
py = jupytext.writes(nb, fmt='py')
Path(path.name).write_text(py)
assert subprocess.call(['ipython', path.name]) == 0