def nbdev_clean_nbs(fname: Param("A notebook name or glob to convert", str) = None, clear_all: Param("Clean all metadata and outputs", bool) = False, disp: Param("Print the cleaned outputs", bool) = False, read_input_stream: Param( "Read input stram and not nb folder") = False): "Clean all notebooks in `fname` to avoid merge conflicts" #Git hooks will pass the notebooks in the stdin if read_input_stream and sys.stdin: input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') nb = json.load(input_stream) clean_nb(nb, clear_all=clear_all) _print_output(nb) return files = Config().nbs_path.glob( '**/*.ipynb') if fname is None else glob.glob(fname) for f in files: if not str(f).endswith('.ipynb'): continue nb = read_nb(f) clean_nb(nb, clear_all=clear_all) if disp: _print_output(nb) else: NotebookNotary().sign(nb) nbformat.write(nb, str(f), version=4)
def nbdev_test_nbs(fname: Param("A notebook name or glob to convert", str) = None, flags: Param("Space separated list of flags", str) = None, n_workers: Param("Number of workers to use", int) = None, verbose: Param("Print errors along the way", bool) = True, timing: Param( "Timing each notebook to see the ones are slow", bool) = True): "Test in parallel the notebooks matching `fname`, passing along `flags`" if flags is not None: flags = flags.split(' ') if fname is None: files = [ f for f in Config().nbs_path.glob('*.ipynb') if not f.name.startswith('_') ] else: files = glob.glob(fname) files = [Path(f).absolute() for f in sorted(files)] if len(files) == 1 and n_workers is None: n_workers = 0 # make sure we are inside the notebook folder of the project os.chdir(Config().nbs_path) results = parallel(_test_one, files, flags=flags, verbose=verbose, n_workers=n_workers) passed, times = [r[0] for r in results], [r[1] for r in results] if all(passed): print("All tests are passing!") else: msg = "The following notebooks failed:\n" raise Exception( msg + '\n'.join([f.name for p, f in zip(passed, files) if not p])) if timing: for i, t in sorted(enumerate(times), key=lambda o: o[1], reverse=True): print(f"Notebook {files[i].name} took {int(t)} seconds")
def fastgpu_poll( path: Param("Path containing `to_run` directory", str) = '.', exit: Param("Exit when `to_run` is empty", int) = 1, ): "Poll `path` for scripts using `ResourcePoolGPU.poll_scripts`" rp = ResourcePoolGPU(path=path) rp.poll_scripts(exit_when_empty=exit)
def nbdev_nb2md( fname: Param("A notebook file name to convert", str), dest: Param("The destination folder", str) = '.', jekyll: Param("To use jekyll metadata for your markdown file or not", bool) = True, ): "Convert the notebook in `fname` to a markdown file" convert_md(fname, dest, jekyll=jekyll)
def nbdev_build_docs(fname:Param("A notebook name or glob to convert", str)=None, force_all:Param("Rebuild even notebooks that haven't changed", bool)=False, mk_readme:Param("Also convert the index notebook to README", bool)=True, n_workers:Param("Number of workers to use", int)=None): "Build the documentation by converting notebooks mathing `fname` to html" notebook2html(fname=fname, force_all=force_all, n_workers=n_workers) if fname is None: make_sidebar() if mk_readme: make_readme()
def nbdev_fix_merge( fname: Param("A notebook filename to fix", str), fast: Param( "Fast fix: automatically fix the merge conflicts in outputs or metadata", bool) = True, trust_us: Param("Use local outputs/metadata when fast mergning", bool) = True): "Fix merge conflicts in notebook `fname`" fix_conflicts(fname, fast=fast, trust_us=trust_us)
def nbdev_trust_nbs(fname:Param("A notebook name or glob to convert", str)=None, force_all:Param("Trust even notebooks that haven't changed", bool)=False): "Trust noteboks matching `fname`" check_fname = Config().nbs_path/".last_checked" last_checked = os.path.getmtime(check_fname) if check_fname.exists() else None files = Config().nbs_path.glob('**/*.ipynb') if fname is None else glob.glob(fname) for fn in files: if last_checked and not force_all: last_changed = os.path.getmtime(fn) if last_changed < last_checked: continue nb = read_nb(fn) if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb) check_fname.touch(exist_ok=True)
def nbdev_read_nbs(fname:Param("A notebook name or glob to convert", str)=None): "Check all notebooks matching `fname` can be opened" files = Config().nbs_path.glob('**/*.ipynb') if fname is None else glob.glob(fname) for nb in files: try: _ = read_nb(nb) except Exception as e: print(f"{nb} is corrupted and can't be opened.") raise e
def nbdev_test_nbs(fname: Param("A notebook name or glob to convert", str) = None, flags: Param("Space separated list of flags", str) = None, n_workers: Param("Number of workers to use", int) = None): "Test in parallel the notebooks matching `fname`, passing along `flags`" if flags is not None: flags = flags.split(' ') if fname is None: files = [ f for f in Config().nbs_path.glob('*.ipynb') if not f.name.startswith('_') ] else: files = glob.glob(fname) files = [Path(f).absolute() for f in files] # make sure we are inside the notebook folder of the project os.chdir(Config().nbs_path) passed = parallel(_test_one, files, flags=flags, n_workers=n_workers) if all(passed): print("All tests are passing!") else: msg = "The following notebooks failed:\n" raise Exception( msg + '\n'.join([f.name for p, f in zip(passed, files) if not p]))
def nbdev_update_lib(fname: Param("A notebook name or glob to convert", str) = None): "Propagates any change in the modules matching `fname` to the notebooks that created them" script2notebook(fname=fname)
def nbdev_build_lib(fname: Param("A notebook name or glob to convert", str) = None): "Export notebooks matching `fname` to python modules" notebook2script(fname=fname) write_tmpls()