def test_clean_relative_subdirectory(notebooks_dir, input_type, organize_by):
""" Test that export works for subdirectory relative to current working directory.
"""
with working_directory(notebooks_dir):
# Set up subdirectory
subdir = Path("subdir")
subdir.mkdir()
for subfile in Path().iterdir():
shutil.move(str(subfile), str(subdir))
sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
expected_notebooks = find_notebooks(subdir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = "subdir"
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(subdir / f"{expected_notebooks[0].path.name}")
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, config))
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(subdir.glob("**/*")) == all_expected
def test_export_relative(notebooks_dir, input_type, organize_by):
""" Test that export works relative to current working directory.
"""
with working_directory(notebooks_dir):
expected_notebooks = find_notebooks(Path())
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE)
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = "."
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = f"{expected_notebooks[0].path.name}"
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, config))
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(Path().glob("**/*")) == all_expected
def test_export_with_config_no_cli_opts(notebooks_dir, input_type, organize_by):
"""Test that export works with a config and no CLI options. Should use config options.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, config))
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_configure_specified(tmp_path):
export_formats = ["script", "html"]
organize_by = "notebook"
clean_exclude = ["README.md", "images/*"]
assert export_formats != DEFAULT_EXPORT_FORMATS
assert organize_by != DEFAULT_ORGANIZE_BY
cmd_list = ["configure", str(tmp_path)]
for fmt in export_formats:
cmd_list.extend(["-f", fmt])
cmd_list.extend(["-b", organize_by])
for excl in clean_exclude:
cmd_list.extend(["-e", excl])
result = CliRunner().invoke(app, cmd_list)
assert result.exit_code == 0
config = NbAutoexportConfig.parse_file(
path=tmp_path / SAVE_PROGRESS_INDICATOR_FILE,
content_type="application/json",
encoding="utf-8",
)
expected_config = NbAutoexportConfig(
export_formats=export_formats,
organize_by=organize_by,
clean=CleanConfig(exclude=clean_exclude),
)
assert config == expected_config
def test_clean(notebooks_dir, need_confirmation, organize_by):
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS,
organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
if need_confirmation:
result = CliRunner().invoke(app, ["clean", str(notebooks_dir)],
input="y")
else:
result = CliRunner().invoke(
app, ["clean", str(notebooks_dir), "--yes"])
assert result.exit_code == 0
expected_notebooks = [
JupyterNotebook.from_file(notebooks_dir / f"{nb}.ipynb")
for nb in EXPECTED_NOTEBOOKS
]
expected_exports = set(get_expected_exports(expected_notebooks, config))
all_expected = {nb.path
for nb in expected_notebooks
} | expected_exports | {sentinel_path}
assert set(notebooks_dir.glob("**/*")) == all_expected
# Run clean again, there should be nothing to do
result_rerun = CliRunner().invoke(app, ["clean", str(notebooks_dir)])
assert result_rerun.exit_code == 0
assert result_rerun.stdout.strip().endswith(
"No files identified for cleaning. Exiting.")
def test_export_with_config_with_cli_opts(notebooks_dir, input_type, organize_by):
"""Test that export works with both config and CLI options. CLI options should overide config.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
written_config = NbAutoexportConfig()
with sentinel_path.open("w") as fp:
fp.write(written_config.json())
expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
assert expected_config != written_config
flags = list(chain(["-b", organize_by], *(["-f", fmt] for fmt in EXPECTED_FORMATS)))
result = CliRunner().invoke(app, ["export", input_path] + flags)
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, expected_config))
expected_exports_from_written = set(get_expected_exports(expected_to_convert, written_config))
assert expected_exports != expected_exports_from_written
all_expected = expected_notebook_files | expected_exports | {sentinel_path}
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_clean_relative(notebooks_dir, organize_by):
""" Test that cleaning works relative to current working directory.
"""
with working_directory(notebooks_dir):
sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE)
config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS,
organize_by=organize_by)
with sentinel_path.open("w") as fp:
fp.write(config.json())
result = CliRunner().invoke(app, ["clean", "."], input="y")
assert result.exit_code == 0
expected_notebooks = [
JupyterNotebook.from_file(f"{nb}.ipynb")
for nb in EXPECTED_NOTEBOOKS
]
expected_exports = set(get_expected_exports(expected_notebooks,
config))
all_expected = {nb.path
for nb in expected_notebooks
} | expected_exports | {sentinel_path}
assert set(Path().glob("**/*")) == all_expected
# Run clean again, there should be nothing to do
result_rerun = CliRunner().invoke(app, ["clean", "."])
assert result_rerun.exit_code == 0
assert result_rerun.stdout.strip().endswith(
"No files identified for cleaning. Exiting.")
def test_clean_exclude(notebooks_dir):
"""Test that cleaning works with exclude"""
with working_directory(notebooks_dir):
# Set up subdirectory
subdir = Path("subdir")
subdir.mkdir()
for subfile in Path().iterdir():
shutil.move(str(subfile), str(subdir))
# Set up extra files
extra_files = [
subdir / "keep.txt",
subdir / "delete.txt",
subdir / "images" / "keep.jpg",
subdir / "images" / "delete.png",
subdir / "pictures" / "delete.jpg",
subdir / "keep.md",
subdir / "docs" / "keep.md",
]
for extra_file in extra_files:
extra_file.parent.mkdir(exist_ok=True)
extra_file.touch()
sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE
config = NbAutoexportConfig(
export_formats=EXPECTED_FORMATS,
organize_by="extension",
clean=CleanConfig(exclude=["keep.txt", "images/*.jpg"]),
)
with sentinel_path.open("w", encoding="utf-8") as fp:
fp.write(config.json())
command = ["clean", "subdir", "-e", "**/*.md"]
result = CliRunner().invoke(app, command, input="y")
assert result.exit_code == 0
expected_notebooks = [
JupyterNotebook.from_file(subdir / f"{nb}.ipynb")
for nb in EXPECTED_NOTEBOOKS
]
expected_exports = set(get_expected_exports(expected_notebooks,
config))
expected_extra = {
extra_file
for extra_file in extra_files if extra_file.match("keep.*")
} | {subdir / "images", subdir / "docs"}
all_expected = ({nb.path
for nb in expected_notebooks}
| expected_exports
| {sentinel_path}
| expected_extra)
assert set(subdir.glob("**/*")) == all_expected
# Run clean again, there should be nothing to do
result_rerun = CliRunner().invoke(app, command)
assert result_rerun.exit_code == 0
assert result_rerun.stdout.strip().endswith(
"No files identified for cleaning. Exiting.")
def test_configure_defaults(tmp_path):
result = CliRunner().invoke(app, ["configure", str(tmp_path)])
assert result.exit_code == 0
config = NbAutoexportConfig.parse_file(
path=tmp_path / SAVE_PROGRESS_INDICATOR_FILE,
content_type="application/json",
encoding="utf-8",
)
expected_config = NbAutoexportConfig()
assert config == expected_config
def post_save(model: dict, os_path: str,
contents_manager: FileContentsManager):
"""Post-save hook for converting notebooks to other formats using Jupyter nbconvert and saving
in a subfolder.
The following arguments are standard for Jupyter post-save hooks. See [Jupyter Documentation](
https://jupyter-notebook.readthedocs.io/en/stable/extending/savehooks.html).
Args:
model (dict): the model representing the file. See [Jupyter documentation](
https://jupyter-notebook.readthedocs.io/en/stable/extending/contents.html#data-model).
os_path (str): the filesystem path to the file just written
contents_manager (FileContentsManager): FileContentsManager instance that hook is bound to
"""
# only do this for notebooks
if model["type"] != "notebook":
return
# only do this if we've added the special indicator file to the working directory
os_path = Path(os_path)
cwd = os_path.parent
save_progress_indicator = cwd / SAVE_PROGRESS_INDICATOR_FILE
should_convert = save_progress_indicator.exists()
if should_convert:
config = NbAutoexportConfig.parse_file(path=save_progress_indicator,
content_type="application/json",
encoding="utf-8")
export_notebook(os_path, config=config)
def test_post_save_organize_by_extension(notebooks_dir):
notebook_path = notebooks_dir / "the_notebook.ipynb"
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
with sentinel_path.open("w") as fp:
json.dump(
NbAutoexportConfig(export_formats=["script", "html"],
organize_by="extension").dict(),
fp,
)
assert notebook_path.exists()
assert sentinel_path.exists()
post_save(model={"type": "notebook"},
os_path=str(notebook_path),
contents_manager=None)
assert set(notebooks_dir.iterdir()) == {
sentinel_path, # sentinel file
notebook_path, # original ipynb
notebooks_dir / "script", # converted notebook directory
notebooks_dir / "html", # converted notebook directory
}
assert (notebooks_dir / "script").is_dir()
assert (notebooks_dir / "html").is_dir()
assert (notebooks_dir / "script" / "the_notebook.py").exists()
assert (notebooks_dir / "html" / "the_notebook.html").exists()
def test_export_notebook(notebooks_dir, organize_by):
"""Test that export notebook works. Explicitly write out expected files, because tests for
get_expected_exports will compare against export_notebook.
"""
notebook = JupyterNotebook.from_file(notebooks_dir / "the_notebook.ipynb")
config = NbAutoexportConfig(export_formats=EXPORT_FORMATS_TO_TEST,
organize_by=organize_by)
export_notebook(notebook.path, config)
expected_exports = set()
for fmt in EXPORT_FORMATS_TO_TEST:
if organize_by == "extension":
subfolder = notebooks_dir / fmt.value
elif organize_by == "notebook":
subfolder = notebooks_dir / notebook.name
extension = get_extension(notebook, fmt)
expected_exports.add(subfolder) # subfolder
expected_exports.add(subfolder /
f"{notebook.name}{extension}") # export file
if fmt in FORMATS_WITH_IMAGE_DIR:
image_subfolder = subfolder / f"{notebook.name}_files"
expected_exports.add(image_subfolder) # image subdir
expected_exports.add(image_subfolder /
f"{notebook.name}_1_1.png") # image file
all_expected = {notebook.path} | expected_exports
assert all_expected.issubset(set(notebooks_dir.glob("**/*")))
def test_find_notebooks(tmp_path, notebook_asset):
shutil.copy(notebook_asset.path, tmp_path / "the_notebook_0.ipynb")
shutil.copy(notebook_asset.path, tmp_path / "the_notebook_1.ipynb")
expected_notebooks = [
JupyterNotebook.from_file(tmp_path / "the_notebook_0.ipynb"),
JupyterNotebook.from_file(tmp_path / "the_notebook_1.ipynb"),
]
# Non-notebook files
(tmp_path / "the_journal.txt").touch()
with (tmp_path / "the_log.json").open("w", encoding="utf-8") as fp:
json.dump(
{
"LOG ENTRY: SOL 61": "How come Aquaman can control whales?",
"LOG ENTRY: SOL 381":
"That makes me a pirate! A space pirate!",
},
fp,
)
with (tmp_path / SAVE_PROGRESS_INDICATOR_FILE).open(
"w", encoding="utf-8") as fp:
fp.write(NbAutoexportConfig().json())
found_notebooks = find_notebooks(tmp_path)
assert set(found_notebooks) == set(expected_notebooks)
def test_export_no_config_with_cli_opts(notebooks_dir, input_type, organize_by):
"""Test export command with no config file and CLI options. Should use CLI options.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
flags = list(chain(["-b", organize_by], *(["-f", fmt] for fmt in EXPECTED_FORMATS)))
result = CliRunner().invoke(app, ["export", input_path] + flags)
assert result.exit_code == 0
assert set(EXPECTED_FORMATS) != set(DEFAULT_EXPORT_FORMATS) # make sure test is meaningful
expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, expected_config))
all_expected = expected_notebook_files | expected_exports
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_invalid_config(file_contents_manager, notebook_file, notebook_model,
caplog):
"""Test that post_save function gracefully logs errors."""
config = NbAutoexportConfig(export_formats=[ExportFormat.script],
organize_by=OrganizeBy.extension)
invalid_config = json.loads(config.json())
invalid_config["export_formats"] = ["triplicate"]
with (notebook_file.parent / SAVE_PROGRESS_INDICATOR_FILE).open(
"w", encoding="utf-8") as fp:
json.dump(invalid_config, fp)
# Runs through, since error is caught
file_contents_manager.save(notebook_model, path=notebook_file.name)
assert caplog_contains(
caplog,
level=logging.ERROR,
in_msg="nbautoexport | post_save failed due to ValidationError",
)
assert not (notebook_file.parent / "script" /
f"{notebook_file.stem}.py").exists()
def test_not_notebook(file_contents_manager, tmp_path):
"""Test that post_save function ignores non-notebook file when FileContentsManager saves."""
config = NbAutoexportConfig(export_formats=[ExportFormat.script],
organize_by=OrganizeBy.extension)
with (tmp_path / SAVE_PROGRESS_INDICATOR_FILE).open(
"w", encoding="utf-8") as fp:
fp.write(config.json())
file_path = tmp_path / "journal.txt"
with file_path.open("w", encoding="utf-8") as fp:
fp.write("I'm a journal.")
model = {
"type": "file",
"format": "text",
"mimetype": "text/plain",
"content": "I'm a journal.",
}
file_contents_manager.save(model, path=str(file_path.name))
assert not (tmp_path / "script" / f"{file_path.stem}.py").exists()
def test_force_overwrite(tmp_path):
(tmp_path / ".nbautoexport").touch()
runner = CliRunner()
result = runner.invoke(app, [
"configure",
str(tmp_path), "-o", "-f", "script", "-f", "html", "-b", "notebook"
])
assert result.exit_code == 0
with (tmp_path / ".nbautoexport").open("r", encoding="utf-8") as fp:
config = json.load(fp)
expected_config = NbAutoexportConfig(export_formats=["script", "html"],
organize_by="notebook")
assert config == expected_config
def test_clean_dry_run(notebooks_dir):
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
with sentinel_path.open("w", encoding="utf-8") as fp:
fp.write(NbAutoexportConfig(export_formats=EXPECTED_FORMATS).json())
starting_files = set(notebooks_dir.glob("**/*"))
result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--dry-run"])
assert result.exit_code == 0
ending_files = set(notebooks_dir.glob("**/*"))
# no files deleted
assert starting_files == ending_files
def test_post_save_type_directory(notebooks_dir):
"""Test that post_save should do nothing if model type is 'directory'."""
notebook_path = notebooks_dir / "the_notebook.ipynb"
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
with sentinel_path.open("w", encoding="utf-8") as fp:
json.dump(NbAutoexportConfig().dict(), fp)
assert notebook_path.exists()
assert sentinel_path.exists()
post_save(model={"type": "directory"}, os_path=str(notebook_path), contents_manager=None)
assert set(notebooks_dir.iterdir()) == {
sentinel_path, # sentinel file
notebook_path, # original ipynb
}
def test_clean_abort(notebooks_dir):
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
with sentinel_path.open("w") as fp:
fp.write(NbAutoexportConfig(export_formats=EXPECTED_FORMATS).json())
starting_files = set(notebooks_dir.glob("**/*"))
result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="n")
assert result.exit_code == 1
assert result.stdout.endswith("Aborted!\n")
ending_files = set(notebooks_dir.glob("**/*"))
# no files deleted
assert starting_files == ending_files
def test_notebook_exports_generator(notebooks_dir, export_format, organize_by):
"""Test that notebook_exports_generator matches what export_notebook produces."""
notebook = find_notebooks(notebooks_dir)[0]
notebook_files = {
notebooks_dir / f"{nb}.ipynb"
for nb in EXPECTED_NOTEBOOKS
}
config = NbAutoexportConfig(export_formats=[export_format],
organize_by=organize_by)
export_notebook(notebook.path, config)
predicted_exports = set(
notebook_exports_generator(notebook, export_format, organize_by))
actual_exports = set(notebooks_dir.glob("**/*")).difference(notebook_files)
assert predicted_exports == actual_exports
def post_save(model: dict, os_path: str,
contents_manager: FileContentsManager):
"""Post-save hook for converting notebooks to other formats using Jupyter nbconvert and saving
in a subfolder.
The following arguments are standard for Jupyter post-save hooks. See [Jupyter Documentation](
https://jupyter-notebook.readthedocs.io/en/stable/extending/savehooks.html).
Args:
model (dict): the model representing the file. See [Jupyter documentation](
https://jupyter-notebook.readthedocs.io/en/stable/extending/contents.html#data-model).
os_path (str): the filesystem path to the file just written
contents_manager (FileContentsManager): FileContentsManager instance that hook is bound to
"""
logger.debug("nbautoexport | Executing nbautoexport.export.post_save ...")
try:
# only do this for notebooks
if model["type"] != "notebook":
logger.debug(
f"nbautoexport | {os_path} is not a notebook. Nothing to do.")
return
# only do this if we've added the special indicator file to the working directory
notebook_path = Path(os_path)
cwd = notebook_path.parent
save_progress_indicator = cwd / SAVE_PROGRESS_INDICATOR_FILE
should_convert = save_progress_indicator.exists()
if should_convert:
logger.info(
f"nbautoexport | {save_progress_indicator} found. Exporting notebook ..."
)
config = NbAutoexportConfig.parse_file(
path=save_progress_indicator,
content_type="application/json",
encoding="utf-8")
export_notebook(notebook_path, config=config)
else:
logger.debug(
f"nbautoexport | {save_progress_indicator} not found. Nothing to do."
)
logger.debug("nbautoexport | post_save successful.")
except Exception as e:
logger.error(
f"nbautoexport | post_save failed due to {type(e).__name__}: {e}")
def test_post_save(file_contents_manager, notebook_file, notebook_model,
caplog):
"""Test that post_save function works when FileContentsManager saves based on config file."""
caplog.set_level(logging.DEBUG)
config = NbAutoexportConfig(export_formats=[ExportFormat.script],
organize_by=OrganizeBy.extension)
with (notebook_file.parent / SAVE_PROGRESS_INDICATOR_FILE).open(
"w", encoding="utf-8") as fp:
fp.write(config.json())
file_contents_manager.save(notebook_model, path=notebook_file.name)
assert caplog_contains(
caplog,
level=logging.INFO,
in_msg=f"nbautoexport | Exporting {notebook_file}",
)
caplog.clear()
assert (notebook_file.parent / "script" /
f"{notebook_file.stem}.py").exists()
assert not (notebook_file.parent / notebook_file.stem /
f"{notebook_file.stem}.html").exists()
# Update config and check that output is different
(notebook_file.parent / "script" / f"{notebook_file.stem}.py").unlink()
config = NbAutoexportConfig(export_formats=[ExportFormat.html],
organize_by=OrganizeBy.notebook)
with (notebook_file.parent / SAVE_PROGRESS_INDICATOR_FILE).open(
"w", encoding="utf-8") as fp:
fp.write(config.json())
file_contents_manager.save(notebook_model, path=notebook_file.name)
assert caplog_contains(
caplog,
level=logging.INFO,
in_msg=f"nbautoexport | Exporting {notebook_file}",
)
caplog.clear()
assert not (notebook_file.parent / "script" /
f"{notebook_file.stem}.py").exists()
assert (notebook_file.parent / notebook_file.stem /
f"{notebook_file.stem}.html").exists()
def test_export_no_config_no_cli_opts(notebooks_dir, input_type):
"""Test export command with no config file and no CLI options. Should use default options.
"""
expected_notebooks = find_notebooks(notebooks_dir)
assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS)
if input_type == "dir":
expected_to_convert = expected_notebooks
input_path = str(notebooks_dir)
elif input_type == "notebook":
expected_to_convert = expected_notebooks[:1]
input_path = str(expected_notebooks[0].path)
result = CliRunner().invoke(app, ["export", input_path])
assert result.exit_code == 0
expected_notebook_files = {nb.path for nb in expected_notebooks}
expected_exports = set(get_expected_exports(expected_to_convert, NbAutoexportConfig()))
all_expected = expected_notebook_files | expected_exports
assert set(notebooks_dir.glob("**/*")) == all_expected
def test_post_save_clean(notebooks_dir):
notebook_path = notebooks_dir / "the_notebook.ipynb"
sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
with sentinel_path.open("w") as fp:
json.dump(
NbAutoexportConfig(export_formats=["script"],
organize_by="extension",
clean=True).dict(),
fp,
)
org_by_notebook_dir = notebooks_dir / "the_notebook"
org_by_notebook_dir.mkdir()
org_by_notebook_script = org_by_notebook_dir / "the_notebook.py"
org_by_notebook_script.touch()
html_dir = notebooks_dir / "html"
html_dir.mkdir()
html_file = html_dir / "the_notebook.html"
html_file.touch()
for path in [
org_by_notebook_dir, org_by_notebook_script, html_dir, html_file
]:
assert path.exists()
post_save(model={"type": "notebook"},
os_path=str(notebook_path),
contents_manager=None)
all_expected = {
notebook_path,
sentinel_path,
notebooks_dir / "script",
notebooks_dir / "script" / "the_notebook.py",
}
assert set(notebooks_dir.glob("**/*")) == all_expected
def clean(
directory: Path = typer.Argument(...,
exists=True,
file_okay=False,
dir_okay=True,
writable=True,
help="Directory to clean."),
yes: bool = typer.Option(
False,
"--yes",
"-y",
help="Assume 'yes' answer to confirmation prompt to delete files."),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Show files that would be removed, without actually removing."),
):
"""Remove subfolders/files not matching .nbautoconvert configuration and existing notebooks.
"""
sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE
validate_sentinel_path(sentinel_path)
config = NbAutoexportConfig.parse_file(path=sentinel_path,
content_type="application/json")
files_to_clean = find_files_to_clean(directory, config)
if len(files_to_clean) == 0:
typer.echo("No files identified for cleaning. Exiting.")
raise typer.Exit(code=0)
typer.echo("Identified following files to clean up:")
for path in sorted(files_to_clean):
typer.echo(f" {path}")
if dry_run:
typer.echo("Dry run completed. Exiting.")
raise typer.Exit(code=0)
if not yes:
typer.confirm("Are you sure you want to delete these files?",
abort=True)
typer.echo("Removing identified files...")
for path in files_to_clean:
if path.is_file():
path.unlink()
# Remove empty subdirectories
typer.echo("Removing empty subdirectories...")
subfolders = (d for d in directory.iterdir() if d.is_dir())
for subfolder in subfolders:
for subsubfolder in subfolder.iterdir():
if subsubfolder.is_dir() and not any(subsubfolder.iterdir()):
typer.echo(f" {subsubfolder}")
subsubfolder.rmdir()
if not any(subfolder.iterdir()):
typer.echo(f" {subfolder}")
subfolder.rmdir()
typer.echo("Cleaning complete.")
def clean(
directory: Path = typer.Argument(
...,
exists=True,
file_okay=False,
dir_okay=True,
writable=True,
help=f"Directory to clean. Must have a {SAVE_PROGRESS_INDICATOR_FILE} config file.",
),
exclude: List[str] = typer.Option(
[],
"--exclude",
"-e",
help=(
"Glob-style patterns that designate files to exclude from deletion. Combined with any "
f"patterns specified in {SAVE_PROGRESS_INDICATOR_FILE} config file."
),
),
yes: bool = typer.Option(
False, "--yes", "-y", help="Assume 'yes' answer to confirmation prompt to delete files."
),
dry_run: bool = typer.Option(
False, "--dry-run", help="Show files that would be removed, without actually removing."
),
):
"""(EXPERIMENTAL) Remove subfolders/files not matching .nbautoexport configuration and
existing notebooks.
Known limitations:
- Not able to correctly handle additional intended files, such as image assets or
non-notebook-related files.
"""
sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE
validate_sentinel_path(sentinel_path)
config = NbAutoexportConfig.parse_file(
path=sentinel_path, content_type="application/json", encoding="utf-8"
)
# Combine exclude patterns from config and command-line
config.clean.exclude.extend(exclude)
if len(config.clean.exclude) > 0:
typer.echo("Excluding files from cleaning using the following patterns:")
for pattern in config.clean.exclude:
typer.echo(f" {pattern}")
files_to_clean = find_files_to_clean(directory, config)
if len(files_to_clean) == 0:
typer.echo("No files identified for cleaning. Exiting.")
raise typer.Exit(code=0)
typer.echo("Identified following files to clean up:")
for path in sorted(files_to_clean):
typer.echo(f" {path}")
if dry_run:
typer.echo("Dry run completed. Exiting.")
raise typer.Exit(code=0)
if not yes:
typer.confirm("Are you sure you want to delete these files?", abort=True)
typer.echo("Removing identified files...")
for path in files_to_clean:
if path.is_file():
path.unlink()
# Remove empty subdirectories
typer.echo("Removing empty subdirectories...")
subfolders = (d for d in directory.iterdir() if d.is_dir())
for subfolder in subfolders:
for subsubfolder in subfolder.iterdir():
if subsubfolder.is_dir() and not any(subsubfolder.iterdir()):
typer.echo(f" {subsubfolder}")
subsubfolder.rmdir()
if not any(subfolder.iterdir()):
typer.echo(f" {subfolder}")
subfolder.rmdir()
typer.echo("Cleaning complete.")
def export(
input: Path = typer.Argument(
...,
exists=True,
file_okay=True,
dir_okay=True,
writable=True,
help="Path to notebook file or directory of notebook files to export.",
),
export_formats: Optional[List[ExportFormat]] = typer.Option(
None,
"--export-format",
"-f",
show_default=True,
help=
("File format(s) to save for each notebook. Multiple formats should be provided using "
"multiple flags, e.g., '-f script -f html -f markdown'. Provided values will override "
"existing .nbautoexport config files. If neither provided, defaults to "
f"{DEFAULT_EXPORT_FORMATS}."),
),
organize_by: Optional[OrganizeBy] = typer.Option(
None,
"--organize-by",
"-b",
show_default=True,
help=
("Whether to save exported file(s) in a subfolder per notebook or per export format. "
"Provided values will override existing .nbautoexport config files. If neither "
f"provided, defaults to '{DEFAULT_ORGANIZE_BY}'."),
),
):
"""Manually export notebook or directory of notebooks.
An .nbautoexport configuration file in same directory as notebook(s) will be used if it
exists. Configuration options specified by command-line options will override configuration
file. If no existing configuration option exists and no values are provided, default values
will be used.
The export command will not do cleaning, regardless of the 'clean' setting in an .nbautoexport
configuration file.
"""
if input.is_dir():
sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE
notebook_paths = [nb.path for nb in find_notebooks(input)]
if len(notebook_paths) == 0:
typer.echo(f"No notebooks found in directory [{input}]. Exiting.")
raise typer.Exit(code=1)
else:
sentinel_path = input.parent / SAVE_PROGRESS_INDICATOR_FILE
notebook_paths = [input]
# Configuration: input options override existing sentinel file
if sentinel_path.exists():
typer.echo(
f"Reading existing configuration file from {sentinel_path} ...")
config = NbAutoexportConfig.parse_file(path=sentinel_path,
content_type="application/json")
# Overrides
if len(export_formats) > 0:
typer.echo(
f"Overriding config with specified export formats: {export_formats}"
)
config.export_formats = export_formats
if organize_by is not None:
typer.echo(
f"Overriding config with specified organization strategy: {export_formats}"
)
config.organize_by = organize_by
else:
typer.echo(
"No configuration found. Using command options as configuration ..."
)
if len(export_formats) == 0:
typer.echo(
f"No export formats specified. Using default: {DEFAULT_EXPORT_FORMATS}"
)
export_formats = DEFAULT_EXPORT_FORMATS
if organize_by is None:
typer.echo(
f"No organize-by specified. Using default: {DEFAULT_ORGANIZE_BY}"
)
organize_by = DEFAULT_ORGANIZE_BY
config = NbAutoexportConfig(export_formats=export_formats,
organize_by=organize_by)
for notebook_path in notebook_paths:
export_notebook(notebook_path, config=config)
def configure(
directory: Path = typer.Argument(
...,
exists=True,
file_okay=False,
dir_okay=True,
writable=True,
help=
"Path to directory of notebook files to configure with nbautoexport.",
),
export_formats: List[ExportFormat] = typer.Option(
DEFAULT_EXPORT_FORMATS,
"--export-format",
"-f",
show_default=True,
help=
("File format(s) to save for each notebook. Multiple formats should be provided using "
"multiple flags, e.g., '-f script -f html -f markdown'."),
),
organize_by: OrganizeBy = typer.Option(
DEFAULT_ORGANIZE_BY,
"--organize-by",
"-b",
show_default=True,
help=
("Whether to save exported file(s) in a subfolder per notebook or per export format. "
),
),
clean: bool = typer.Option(
DEFAULT_CLEAN,
show_default=True,
help=
"Whether to automatically delete files that don't match expected exports given "
+ "notebooks and configuration.",
),
overwrite: bool = typer.Option(
False,
"--overwrite",
"-o",
is_flag=True,
show_default=True,
help="Overwrite existing configuration, if one is detected.",
),
verbose: bool = typer.Option(False,
"--verbose",
"-v",
is_flag=True,
show_default=True,
help="Verbose mode"),
):
"""
Create a .nbautoexport configuration file in a directory. If nbautoexport has been installed
with the 'install' command, then Jupyter will automatically export any notebooks on save that
are in the same directory as the .nbautoexport file.
An .nbautoexport configuration file only applies to that directory, nonrecursively. You must
independently configure other directories containing notebooks.
"""
if verbose:
logging.basicConfig(level=logging.DEBUG)
config = NbAutoexportConfig(export_formats=export_formats,
organize_by=organize_by,
clean=clean)
try:
install_sentinel(directory=directory,
config=config,
overwrite=overwrite)
except FileExistsError as msg:
typer.echo(msg)
raise typer.Exit(code=1)
# Check for installation in Jupyter config
installed = False
jupyter_config_file = (
(Path(jupyter_config_dir()) /
"jupyter_notebook_config.py").expanduser().resolve())
if jupyter_config_file.exists():
with jupyter_config_file.open("r") as fp:
jupyter_config_text = fp.read()
if block_regex.search(jupyter_config_text):
installed = True
version_match = version_regex.search(jupyter_config_text)
if version_match:
existing_version = version_match.group()
else:
existing_version = ""
if parse_version(existing_version) < parse_version(__version__):
typer.echo(
"Warning: nbautoexport initialize is an older version. "
"Please run 'install' command to update.")
if not installed:
typer.echo(
"Warning: nbautoexport is not properly installed with Jupyter. "
"Please run 'install' command.")
