def nbdev_trust(
fname: str = None, # A notebook name or glob to trust
force_all: bool = False # Also trust notebooks that haven't changed
):
"Trust notebooks matching `fname`"
try:
from nbformat.sign import NotebookNotary
except:
import warnings
warnings.warn("Please install jupyter and try again")
return
fname = Path(fname if fname else config_key("nbs_path", '.'))
path = fname if fname.is_dir() else fname.parent
check_fname = path / ".last_checked"
last_checked = os.path.getmtime(
check_fname) if check_fname.exists() else None
nbs = globtastic(fname, file_glob='*.ipynb',
skip_folder_re='^[_.]') if fname.is_dir() else [fname]
for fn in nbs:
if last_checked and not force_all:
last_changed = os.path.getmtime(fn)
if last_changed < last_checked: continue
nb = read_nb(fn)
if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb)
check_fname.touch(exist_ok=True)
def nbdev_trust_nbs(fname:Param("A notebook name or glob to convert", str)=None,
force_all:Param("Trust even notebooks that haven't changed", bool)=False):
"Trust notebooks matching `fname`"
check_fname = Config().path("nbs_path")/".last_checked"
last_checked = os.path.getmtime(check_fname) if check_fname.exists() else None
files = Config().path("nbs_path").glob('**/*.ipynb') if fname is None else glob.glob(fname)
for fn in files:
if last_checked and not force_all:
last_changed = os.path.getmtime(fn)
if last_changed < last_checked: continue
nb = read_nb(fn)
if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb)
check_fname.touch(exist_ok=True)
def link_nb(nb_path):
nb = read_nb(nb_path)
cells = nb['cells']
link_markdown_cells(cells, get_imported_modules(cells))
add_nb_metadata(nb, nb_path)
json.dump(nb, open(nb_path, 'w'))
NotebookNotary().sign(read_nb(nb_path))
def _script2notebook(fname, dic, silent=False):
"Put the content of `fname` back in the notebooks it came from."
if os.environ.get('IN_TEST', 0): return # don't export if running tests
fname = Path(fname)
with open(fname, encoding='utf8') as f:
code = f.read()
splits = _split(code)
rel_name = fname.absolute().resolve().relative_to(
Config().path("lib_path"))
key = str(rel_name.with_suffix(''))
assert len(splits) == len(
dic[key]
), f'"{rel_name}" exported from notebooks should have {len(dic[key])} cells but has {len(splits)}.'
assert all([c1[0] == c2[1]] for c1, c2 in zip(splits, dic[key]))
splits = [(c2[0], c1[0], c1[1]) for c1, c2 in zip(splits, dic[key])]
nb_fnames = {Config().path("nbs_path") / s[1] for s in splits}
for nb_fname in nb_fnames:
nb = read_nb(nb_fname)
for i, f, c in splits:
c = _deal_loc_import(c, str(fname))
if f == nb_fname.name:
flags = split_flags_and_code(nb['cells'][i], str)[0]
nb['cells'][i]['source'] = flags + '\n' + c.replace('', '')
NotebookNotary().sign(nb)
nbformat.write(nb, str(nb_fname), version=4)
if not silent: print(f"Converted {rel_name}.")
def create_notebook(template, notebook_dir, args, kargs):
all_args = ' '.join(args)
if all_args and kargs: all_args += ' '
all_args += ' '.join(kargs[key] for key in sorted(kargs.keys()))
all_args = sub(r'\s+', '-', all_args)
vars = dict(kargs)
for name, value in zip(getfullargspec(template).args, args):
vars[name] = value
template = template.__name__
base = join(notebook_dir, template)
all_args = all_args if all_args else template
name = all_args + IPYNB
path = join(base, name)
makedirs(base, exist_ok=True)
log.info(f'Creating {template} at {path} with {vars}')
notebook = load_notebook(template, vars)
# https://testnb.readthedocs.io/en/latest/security.html
NotebookNotary().sign(notebook)
if exists(path):
log.warning(f'Deleting old version of {path}')
unlink(path)
with open(path, 'w') as out:
log.info(f'Writing {template} to {path}')
nb.write(notebook, out)
return join(template, name)
def link_nb(nb_path):
nb = read_nb(nb_path)
cells = nb['cells']
link_markdown_cells(cells, get_imported_modules(cells))
add_nb_metadata(nb, nb_path)
write_nb(nb, nb_path)
NotebookNotary().sign(read_nb(nb_path))
def nbdev_trust_nbs(
fname: str = None, # A notebook name or glob to convert
force_all: bool = False # Trust even notebooks that havent changed
):
"Trust notebooks matching `fname`"
check_fname = get_config().path("nbs_path") / ".last_checked"
last_checked = os.path.getmtime(
check_fname) if check_fname.exists() else None
files = nbglob(fname)
for fn in files:
if last_checked and not force_all:
last_changed = os.path.getmtime(fn)
if last_changed < last_checked: continue
nb = read_nb(fn)
if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb)
check_fname.touch(exist_ok=True)
def update_nb_metadata(nb_path=None, title=None, summary=None, keywords='fastai', overwrite=True, **kwargs):
"Creates jekyll metadata for given notebook path."
nb = read_nb(nb_path)
data = {'title': title, 'summary': summary, 'keywords': keywords, **kwargs}
data = {k:v for (k,v) in data.items() if v is not None} # remove none values
if not data: return
nb['metadata']['jekyll'] = data
write_nb(nb, nb_path)
NotebookNotary().sign(nb)
def execute_nb(fname, metadata=None):
"Execute notebook `fname` with `metadata` for preprocessing."
# Any module used in the notebook that isn't inside must be in the same directory as this script
with open(fname) as f: nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
metadata = metadata or {}
ep.preprocess(nb, metadata)
with open(fname, 'wt') as f: nbformat.write(nb, f)
NotebookNotary().sign(nb)
def execute_nb(fname, metadata=None, save=True, show_doc_only=False):
"Execute notebook `fname` with `metadata` for preprocessing."
# Any module used in the notebook that isn't inside must be in the same directory as this script
with open(fname) as f: nb = nbformat.read(f, as_version=4)
ep_class = ExecuteShowDocPreprocessor if show_doc_only else ExecutePreprocessor
ep = ep_class(timeout=600, kernel_name='python3')
metadata = metadata or {}
ep.preprocess(nb, metadata)
if save:
with open(fname, 'wt') as f: nbformat.write(nb, f)
NotebookNotary().sign(nb)
def update_nb_metadata(
nb_path=None, title=None, summary=None, keywords="fastai", overwrite=True, **kwargs
):
"Creates jekyll metadata for given notebook path."
nb = read_nb(nb_path)
data = {"title": title, "summary": summary, "keywords": keywords, **kwargs}
data = {k: v for (k, v) in data.items() if v is not None} # remove none values
if not data:
return
nb["metadata"]["jekyll"] = data
write_nb(nb, nb_path)
NotebookNotary().sign(nb)
def update_nb_metadata(nb_path=None,
title=None,
summary=None,
keywords='fastai',
overwrite=True,
**kwargs):
"Creates jekyll metadata for given notebook path."
nb = read_nb(nb_path)
jm = {'title': title, 'summary': summary, 'keywords': keywords, **kwargs}
update_metadata(nb, jm, overwrite)
write_nb(nb, nb_path)
NotebookNotary().sign(nb)
def _script2notebook(fname, silent=False):
"Put the content of `fname` back in the notebooks it came from."
if os.environ.get('IN_TEST',0): return # don't export if running tests
fname = Path(fname)
with open(fname) as f: code = f.read()
splits = _split(code)
nb_fnames = {s[1] for s in splits}
for nb_fname in nb_fnames:
nb = read_nb(nb_fname)
for i,f,c in splits:
c = _deal_loc_import(c, str(fname))
if f == nb_fname:
l = nb['cells'][i]['source'].split('\n')[0]
nb['cells'][i]['source'] = l + '\n' + c
NotebookNotary().sign(nb)
nbformat.write(nb, nb_fname, version=4)
if not silent: print(f"Converted {fname}.")
def _script2notebook(fname, dic, silent=False):
"Put the content of `fname` back in the notebooks it came from."
if os.environ.get('IN_TEST',0): return # don't export if running tests
fname = Path(fname)
with open(fname) as f: code = f.read()
splits = _split(code)
assert len(splits) == len(dic[fname]), f"Exported file from notebooks should have {len(dic[fname])} cells but has {len(splits)}."
assert np.all([c1[0]==c2[1]] for c1,c2 in zip(splits, dic[fname]))
splits = [(c2[0],c1[0],c1[1]) for c1,c2 in zip(splits, dic[fname])]
nb_fnames = {s[1] for s in splits}
for nb_fname in nb_fnames:
nb = read_nb(nb_fname)
for i,f,c in splits:
c = _deal_loc_import(c, str(fname))
if f == nb_fname:
l = nb['cells'][i]['source'].split('\n')[0]
nb['cells'][i]['source'] = l + '\n' + c
NotebookNotary().sign(nb)
nbformat.write(nb, nb_fname, version=4)
if not silent: print(f"Converted {fname}.")
def create_notebook(template, base, args, kargs):
if hasattr(template, '_original'): # drop wrapper
template = template._original
all_args = ' '.join(args)
if all_args and kargs: all_args += ' '
all_args += ' '.join(kargs[key] for key in sorted(kargs.keys()))
all_args = sub(r'\s+', '-', all_args)
all_args = sub(escape(sep), '-', all_args)
vars = dict(kargs)
vars[BASE] = base
spec = getfullargspec(template)
for name, value in zip_longest(spec.args, args):
if name:
vars[name] = value
else:
if spec.varargs not in vars:
vars[spec.varargs] = []
vars[spec.varargs].append(value)
template = template.__name__
notebook_dir = base_system_path(base, subdir=NOTEBOOKS)
root = join(notebook_dir, template)
all_args = all_args if all_args else template
name = all_args + IPYNB
path = join(root, name)
makedirs(root, exist_ok=True)
log.info(f'Creating {template} at {path} with {vars}')
notebook = load_notebook(template, vars)
# https://testnb.readthedocs.io/en/latest/security.html
NotebookNotary().sign(notebook)
if exists(path):
log.debug(f'Deleting old version of {path}')
unlink(path)
with open(path, 'w') as out:
log.info(f'Writing {template} to {path}')
nb.write(notebook, out)
return join(template, name)
def create_notebook(config, template, args):
if hasattr(template, '_original'): # drop wrapper
template = template._original
all_args = ' '.join(args)
all_args = sub(r'\s+', '-', all_args)
all_args = sub(escape(sep), '-', all_args)
vars = {BASE: config.args[BASE], URI: config.args._format(name=URI)}
spec = getfullargspec(template)
for name, value in zip_longest(spec.args, args):
if name:
vars[name] = value
else:
if spec.varargs not in vars:
vars[spec.varargs] = []
vars[spec.varargs].append(value)
template = template.__name__
notebook_dir = config.args._format_path(NOTEBOOK_DIR)
root = join(notebook_dir, template)
all_args = all_args if all_args else template
name = all_args + IPYNB
path = join(root, name)
makedirs(root, exist_ok=True)
log.info(f'Creating {template} at {path} with {vars}')
notebook = load_notebook(template, vars)
# https://testnb.readthedocs.io/en/latest/security.html
log.debug(f'Notary using DB at {notebook_dir}')
NotebookNotary(data_dir=notebook_dir).sign(notebook)
if exists(path):
log.debug(f'Deleting old version of {path}')
unlink(path)
with open(path, 'w') as out:
log.info(f'Writing {template} to {path}')
nb.write(notebook, out)
return join(template, name)
def link_nb(nb_path):
nb = read_nb(nb_path)
cells = nb['cells']
link_markdown_cells(cells, get_imported_modules(cells, Path(nb_path).stem))
write_nb(nb, nb_path)
NotebookNotary().sign(read_nb(nb_path))
def preprocess(self, nb, resources): NotebookNotary().sign(nb) return nb, resources
import os
import re
import traceback
from collections import UserString
from typing import List, TextIO
import nbformat
from nbformat.sign import NotebookNotary
from nbformat.notebooknode import NotebookNode, from_dict
notebook_notary = NotebookNotary()
class TestCell(UserString):
"""
A class for Jupyter Notebook code cell.
Attributes
----------
ignore : bool
Whether the cell magic line contained the `-t` option.
name : str
The user defined name of the test cell block.
"""
def __init__(self, data, notebook):
super().__init__(data["source"])
magic_args = self.parse_cell_magic(self)
if "-n" in magic_args:
self.ignore = True
magic_args.remove("-n")
else:
def sign(self):
notary = Notary()
signature = notary.compute_signature(self.notebook)
if not signature.startswith("sha256:"):
signature = "sha256:" + signature
self.notebook['metadata']['signature'] = signature