def test_find_notebooks(tmp_path, notebook_asset):
shutil.copy(notebook_asset.path, tmp_path / "the_notebook_0.ipynb")
shutil.copy(notebook_asset.path, tmp_path / "the_notebook_1.ipynb")
expected_notebooks = [
JupyterNotebook.from_file(tmp_path / "the_notebook_0.ipynb"),
JupyterNotebook.from_file(tmp_path / "the_notebook_1.ipynb"),
]
# Non-notebook files
(tmp_path / "the_journal.txt").touch()
with (tmp_path / "the_log.json").open("w", encoding="utf-8") as fp:
json.dump(
{
"LOG ENTRY: SOL 61": "How come Aquaman can control whales?",
"LOG ENTRY: SOL 381":
"That makes me a pirate! A space pirate!",
},
fp,
)
with (tmp_path / SAVE_PROGRESS_INDICATOR_FILE).open(
"w", encoding="utf-8") as fp:
fp.write(NbAutoexportConfig().json())
found_notebooks = find_notebooks(tmp_path)
assert set(found_notebooks) == set(expected_notebooks)
def test_export_no_config_with_cli_opts(notebooks_dir, input_type, organize_by):
"""Test export command with no config file and CLI options. Should use CLI options.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
flags = list(chain(["-b", organize_by], *(["-f", fmt] for fmt in EXPECTED_FORMATS)))
result = CliRunner().invoke(app, ["export", input_path] + flags)
assert result.exit_code == 0
assert set(EXPECTED_FORMATS) != set(DEFAULT_EXPORT_FORMATS) # make sure test is meaningful
expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, expected_config))
all_expected = expected_notebook_files | expected_exports
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_export_with_config_no_cli_opts(notebooks_dir, input_type, organize_by):
"""Test that export works with a config and no CLI options. Should use config options.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, config))
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_clean_relative_subdirectory(notebooks_dir, input_type, organize_by):
""" Test that export works for subdirectory relative to current working directory.
"""
with working_directory(notebooks_dir):
# Set up subdirectory
subdir = Path("subdir")
subdir.mkdir()
for subfile in Path().iterdir():
shutil.move(str(subfile), str(subdir))
sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
expected_notebooks = find_notebooks(subdir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = "subdir"
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(subdir / f"{expected_notebooks[0].path.name}")
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, config))
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(subdir.glob("**/*")) == all_expected
def test_export_relative(notebooks_dir, input_type, organize_by):
""" Test that export works relative to current working directory.
"""
with working_directory(notebooks_dir):
expected_notebooks = find_notebooks(Path())
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE)
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = "."
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = f"{expected_notebooks[0].path.name}"
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, config))
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(Path().glob("**/*")) == all_expected
def test_export_with_config_with_cli_opts(notebooks_dir, input_type, organize_by):
"""Test that export works with both config and CLI options. CLI options should overide config.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
written_config = NbAutoexportConfig()
with sentinel_path.open("w") as fp:
fp.write(written_config.json())
expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
assert expected_config != written_config
flags = list(chain(["-b", organize_by], *(["-f", fmt] for fmt in EXPECTED_FORMATS)))
result = CliRunner().invoke(app, ["export", input_path] + flags)
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, expected_config))
expected_exports_from_written = set(get_expected_exports(expected_to_convert, written_config))
assert expected_exports != expected_exports_from_written
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(notebooks_dir.glob("**/*")) == all_expected
def find_files_to_clean(directory: Path,
config: NbAutoexportConfig) -> List[Path]:
"""Given path to a notebooks directory watched by nbautoexport, find all files that are not
expected exports by current nbautoexport configuration and existing notebooks, or other
expected Jupyter or nbautoexport files.
Args:
directory (Path): notebooks directory to find files to clean up
Returns:
List[Path]: list of files to clean up
"""
notebooks: List[JupyterNotebook] = find_notebooks(directory)
expected_exports: List[Path] = get_expected_exports(notebooks, config)
checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*")
if f.is_file())
sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE
subfiles = (f for f in directory.glob("**/*") if f.is_file())
to_clean = (set(subfiles).difference(
nb.path for nb in notebooks).difference(expected_exports).difference(
globs(directory=directory,
patterns=config.clean.exclude)).difference(
checkpoints).difference([sentinel_path]))
return sorted(to_clean)
def test_notebook_exports_generator(notebooks_dir, export_format, organize_by):
"""Test that notebook_exports_generator matches what export_notebook produces."""
notebook = find_notebooks(notebooks_dir)[0]
notebook_files = {
notebooks_dir / f"{nb}.ipynb"
for nb in EXPECTED_NOTEBOOKS
}
config = NbAutoexportConfig(export_formats=[export_format],
organize_by=organize_by)
export_notebook(notebook.path, config)
predicted_exports = set(
notebook_exports_generator(notebook, export_format, organize_by))
actual_exports = set(notebooks_dir.glob("**/*")).difference(notebook_files)
assert predicted_exports == actual_exports
def test_export_no_config_no_cli_opts(notebooks_dir, input_type):
"""Test export command with no config file and no CLI options. Should use default options.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, NbAutoexportConfig()))
all_expected = expected_notebook_files | expected_exports
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_find_notebooks_warning(tmp_path):
bad_notebook_path = tmp_path / "the_journal.ipynb"
bad_notebook_path.touch()
with pytest.warns(Warning, match="Error reading"):
find_notebooks(tmp_path)
def export(
input: Path = typer.Argument(
...,
exists=True,
file_okay=True,
dir_okay=True,
writable=True,
help="Path to notebook file or directory of notebook files to export.",
),
export_formats: Optional[List[ExportFormat]] = typer.Option(
None,
"--export-format",
"-f",
show_default=True,
help=
("File format(s) to save for each notebook. Multiple formats should be provided using "
"multiple flags, e.g., '-f script -f html -f markdown'. Provided values will override "
"existing .nbautoexport config files. If neither provided, defaults to "
f"{DEFAULT_EXPORT_FORMATS}."),
),
organize_by: Optional[OrganizeBy] = typer.Option(
None,
"--organize-by",
"-b",
show_default=True,
help=
("Whether to save exported file(s) in a subfolder per notebook or per export format. "
"Provided values will override existing .nbautoexport config files. If neither "
f"provided, defaults to '{DEFAULT_ORGANIZE_BY}'."),
),
):
"""Manually export notebook or directory of notebooks.
An .nbautoexport configuration file in same directory as notebook(s) will be used if it
exists. Configuration options specified by command-line options will override configuration
file. If no existing configuration option exists and no values are provided, default values
will be used.
The export command will not do cleaning, regardless of the 'clean' setting in an .nbautoexport
configuration file.
"""
if input.is_dir():
sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE
notebook_paths = [nb.path for nb in find_notebooks(input)]
if len(notebook_paths) == 0:
typer.echo(f"No notebooks found in directory [{input}]. Exiting.")
raise typer.Exit(code=1)
else:
sentinel_path = input.parent / SAVE_PROGRESS_INDICATOR_FILE
notebook_paths = [input]
# Configuration: input options override existing sentinel file
if sentinel_path.exists():
typer.echo(
f"Reading existing configuration file from {sentinel_path} ...")
config = NbAutoexportConfig.parse_file(path=sentinel_path,
content_type="application/json")
# Overrides
if len(export_formats) > 0:
typer.echo(
f"Overriding config with specified export formats: {export_formats}"
)
config.export_formats = export_formats
if organize_by is not None:
typer.echo(
f"Overriding config with specified organization strategy: {export_formats}"
)
config.organize_by = organize_by
else:
typer.echo(
"No configuration found. Using command options as configuration ..."
)
if len(export_formats) == 0:
typer.echo(
f"No export formats specified. Using default: {DEFAULT_EXPORT_FORMATS}"
)
export_formats = DEFAULT_EXPORT_FORMATS
if organize_by is None:
typer.echo(
f"No organize-by specified. Using default: {DEFAULT_ORGANIZE_BY}"
)
organize_by = DEFAULT_ORGANIZE_BY
config = NbAutoexportConfig(export_formats=export_formats,
organize_by=organize_by)
for notebook_path in notebook_paths:
export_notebook(notebook_path, config=config)
