def nbdev_trust( fname: str = None, # A notebook name or glob to trust force_all: bool = False # Also trust notebooks that haven't changed ): "Trust notebooks matching `fname`" try: from nbformat.sign import NotebookNotary except: import warnings warnings.warn("Please install jupyter and try again") return fname = Path(fname if fname else config_key("nbs_path", '.')) path = fname if fname.is_dir() else fname.parent check_fname = path / ".last_checked" last_checked = os.path.getmtime( check_fname) if check_fname.exists() else None nbs = globtastic(fname, file_glob='*.ipynb', skip_folder_re='^[_.]') if fname.is_dir() else [fname] for fn in nbs: if last_checked and not force_all: last_changed = os.path.getmtime(fn) if last_changed < last_checked: continue nb = read_nb(fn) if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb) check_fname.touch(exist_ok=True)
def nbdev_trust_nbs(fname:Param("A notebook name or glob to convert", str)=None, force_all:Param("Trust even notebooks that haven't changed", bool)=False): "Trust notebooks matching `fname`" check_fname = Config().path("nbs_path")/".last_checked" last_checked = os.path.getmtime(check_fname) if check_fname.exists() else None files = Config().path("nbs_path").glob('**/*.ipynb') if fname is None else glob.glob(fname) for fn in files: if last_checked and not force_all: last_changed = os.path.getmtime(fn) if last_changed < last_checked: continue nb = read_nb(fn) if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb) check_fname.touch(exist_ok=True)
def link_nb(nb_path): nb = read_nb(nb_path) cells = nb['cells'] link_markdown_cells(cells, get_imported_modules(cells)) add_nb_metadata(nb, nb_path) json.dump(nb, open(nb_path, 'w')) NotebookNotary().sign(read_nb(nb_path))
def _script2notebook(fname, dic, silent=False): "Put the content of `fname` back in the notebooks it came from." if os.environ.get('IN_TEST', 0): return # don't export if running tests fname = Path(fname) with open(fname, encoding='utf8') as f: code = f.read() splits = _split(code) rel_name = fname.absolute().resolve().relative_to( Config().path("lib_path")) key = str(rel_name.with_suffix('')) assert len(splits) == len( dic[key] ), f'"{rel_name}" exported from notebooks should have {len(dic[key])} cells but has {len(splits)}.' assert all([c1[0] == c2[1]] for c1, c2 in zip(splits, dic[key])) splits = [(c2[0], c1[0], c1[1]) for c1, c2 in zip(splits, dic[key])] nb_fnames = {Config().path("nbs_path") / s[1] for s in splits} for nb_fname in nb_fnames: nb = read_nb(nb_fname) for i, f, c in splits: c = _deal_loc_import(c, str(fname)) if f == nb_fname.name: flags = split_flags_and_code(nb['cells'][i], str)[0] nb['cells'][i]['source'] = flags + '\n' + c.replace('', '') NotebookNotary().sign(nb) nbformat.write(nb, str(nb_fname), version=4) if not silent: print(f"Converted {rel_name}.")
def create_notebook(template, notebook_dir, args, kargs): all_args = ' '.join(args) if all_args and kargs: all_args += ' ' all_args += ' '.join(kargs[key] for key in sorted(kargs.keys())) all_args = sub(r'\s+', '-', all_args) vars = dict(kargs) for name, value in zip(getfullargspec(template).args, args): vars[name] = value template = template.__name__ base = join(notebook_dir, template) all_args = all_args if all_args else template name = all_args + IPYNB path = join(base, name) makedirs(base, exist_ok=True) log.info(f'Creating {template} at {path} with {vars}') notebook = load_notebook(template, vars) # https://testnb.readthedocs.io/en/latest/security.html NotebookNotary().sign(notebook) if exists(path): log.warning(f'Deleting old version of {path}') unlink(path) with open(path, 'w') as out: log.info(f'Writing {template} to {path}') nb.write(notebook, out) return join(template, name)
def link_nb(nb_path): nb = read_nb(nb_path) cells = nb['cells'] link_markdown_cells(cells, get_imported_modules(cells)) add_nb_metadata(nb, nb_path) write_nb(nb, nb_path) NotebookNotary().sign(read_nb(nb_path))
def nbdev_trust_nbs( fname: str = None, # A notebook name or glob to convert force_all: bool = False # Trust even notebooks that havent changed ): "Trust notebooks matching `fname`" check_fname = get_config().path("nbs_path") / ".last_checked" last_checked = os.path.getmtime( check_fname) if check_fname.exists() else None files = nbglob(fname) for fn in files: if last_checked and not force_all: last_changed = os.path.getmtime(fn) if last_changed < last_checked: continue nb = read_nb(fn) if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb) check_fname.touch(exist_ok=True)
def update_nb_metadata(nb_path=None, title=None, summary=None, keywords='fastai', overwrite=True, **kwargs): "Creates jekyll metadata for given notebook path." nb = read_nb(nb_path) data = {'title': title, 'summary': summary, 'keywords': keywords, **kwargs} data = {k:v for (k,v) in data.items() if v is not None} # remove none values if not data: return nb['metadata']['jekyll'] = data write_nb(nb, nb_path) NotebookNotary().sign(nb)
def execute_nb(fname, metadata=None): "Execute notebook `fname` with `metadata` for preprocessing." # Any module used in the notebook that isn't inside must be in the same directory as this script with open(fname) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name='python3') metadata = metadata or {} ep.preprocess(nb, metadata) with open(fname, 'wt') as f: nbformat.write(nb, f) NotebookNotary().sign(nb)
def execute_nb(fname, metadata=None, save=True, show_doc_only=False): "Execute notebook `fname` with `metadata` for preprocessing." # Any module used in the notebook that isn't inside must be in the same directory as this script with open(fname) as f: nb = nbformat.read(f, as_version=4) ep_class = ExecuteShowDocPreprocessor if show_doc_only else ExecutePreprocessor ep = ep_class(timeout=600, kernel_name='python3') metadata = metadata or {} ep.preprocess(nb, metadata) if save: with open(fname, 'wt') as f: nbformat.write(nb, f) NotebookNotary().sign(nb)
def update_nb_metadata( nb_path=None, title=None, summary=None, keywords="fastai", overwrite=True, **kwargs ): "Creates jekyll metadata for given notebook path." nb = read_nb(nb_path) data = {"title": title, "summary": summary, "keywords": keywords, **kwargs} data = {k: v for (k, v) in data.items() if v is not None} # remove none values if not data: return nb["metadata"]["jekyll"] = data write_nb(nb, nb_path) NotebookNotary().sign(nb)
def update_nb_metadata(nb_path=None, title=None, summary=None, keywords='fastai', overwrite=True, **kwargs): "Creates jekyll metadata for given notebook path." nb = read_nb(nb_path) jm = {'title': title, 'summary': summary, 'keywords': keywords, **kwargs} update_metadata(nb, jm, overwrite) write_nb(nb, nb_path) NotebookNotary().sign(nb)
def _script2notebook(fname, silent=False): "Put the content of `fname` back in the notebooks it came from." if os.environ.get('IN_TEST',0): return # don't export if running tests fname = Path(fname) with open(fname) as f: code = f.read() splits = _split(code) nb_fnames = {s[1] for s in splits} for nb_fname in nb_fnames: nb = read_nb(nb_fname) for i,f,c in splits: c = _deal_loc_import(c, str(fname)) if f == nb_fname: l = nb['cells'][i]['source'].split('\n')[0] nb['cells'][i]['source'] = l + '\n' + c NotebookNotary().sign(nb) nbformat.write(nb, nb_fname, version=4) if not silent: print(f"Converted {fname}.")
def _script2notebook(fname, dic, silent=False): "Put the content of `fname` back in the notebooks it came from." if os.environ.get('IN_TEST',0): return # don't export if running tests fname = Path(fname) with open(fname) as f: code = f.read() splits = _split(code) assert len(splits) == len(dic[fname]), f"Exported file from notebooks should have {len(dic[fname])} cells but has {len(splits)}." assert np.all([c1[0]==c2[1]] for c1,c2 in zip(splits, dic[fname])) splits = [(c2[0],c1[0],c1[1]) for c1,c2 in zip(splits, dic[fname])] nb_fnames = {s[1] for s in splits} for nb_fname in nb_fnames: nb = read_nb(nb_fname) for i,f,c in splits: c = _deal_loc_import(c, str(fname)) if f == nb_fname: l = nb['cells'][i]['source'].split('\n')[0] nb['cells'][i]['source'] = l + '\n' + c NotebookNotary().sign(nb) nbformat.write(nb, nb_fname, version=4) if not silent: print(f"Converted {fname}.")
def create_notebook(template, base, args, kargs): if hasattr(template, '_original'): # drop wrapper template = template._original all_args = ' '.join(args) if all_args and kargs: all_args += ' ' all_args += ' '.join(kargs[key] for key in sorted(kargs.keys())) all_args = sub(r'\s+', '-', all_args) all_args = sub(escape(sep), '-', all_args) vars = dict(kargs) vars[BASE] = base spec = getfullargspec(template) for name, value in zip_longest(spec.args, args): if name: vars[name] = value else: if spec.varargs not in vars: vars[spec.varargs] = [] vars[spec.varargs].append(value) template = template.__name__ notebook_dir = base_system_path(base, subdir=NOTEBOOKS) root = join(notebook_dir, template) all_args = all_args if all_args else template name = all_args + IPYNB path = join(root, name) makedirs(root, exist_ok=True) log.info(f'Creating {template} at {path} with {vars}') notebook = load_notebook(template, vars) # https://testnb.readthedocs.io/en/latest/security.html NotebookNotary().sign(notebook) if exists(path): log.debug(f'Deleting old version of {path}') unlink(path) with open(path, 'w') as out: log.info(f'Writing {template} to {path}') nb.write(notebook, out) return join(template, name)
def create_notebook(config, template, args): if hasattr(template, '_original'): # drop wrapper template = template._original all_args = ' '.join(args) all_args = sub(r'\s+', '-', all_args) all_args = sub(escape(sep), '-', all_args) vars = {BASE: config.args[BASE], URI: config.args._format(name=URI)} spec = getfullargspec(template) for name, value in zip_longest(spec.args, args): if name: vars[name] = value else: if spec.varargs not in vars: vars[spec.varargs] = [] vars[spec.varargs].append(value) template = template.__name__ notebook_dir = config.args._format_path(NOTEBOOK_DIR) root = join(notebook_dir, template) all_args = all_args if all_args else template name = all_args + IPYNB path = join(root, name) makedirs(root, exist_ok=True) log.info(f'Creating {template} at {path} with {vars}') notebook = load_notebook(template, vars) # https://testnb.readthedocs.io/en/latest/security.html log.debug(f'Notary using DB at {notebook_dir}') NotebookNotary(data_dir=notebook_dir).sign(notebook) if exists(path): log.debug(f'Deleting old version of {path}') unlink(path) with open(path, 'w') as out: log.info(f'Writing {template} to {path}') nb.write(notebook, out) return join(template, name)
def link_nb(nb_path): nb = read_nb(nb_path) cells = nb['cells'] link_markdown_cells(cells, get_imported_modules(cells, Path(nb_path).stem)) write_nb(nb, nb_path) NotebookNotary().sign(read_nb(nb_path))
def preprocess(self, nb, resources): NotebookNotary().sign(nb) return nb, resources
import os import re import traceback from collections import UserString from typing import List, TextIO import nbformat from nbformat.sign import NotebookNotary from nbformat.notebooknode import NotebookNode, from_dict notebook_notary = NotebookNotary() class TestCell(UserString): """ A class for Jupyter Notebook code cell. Attributes ---------- ignore : bool Whether the cell magic line contained the `-t` option. name : str The user defined name of the test cell block. """ def __init__(self, data, notebook): super().__init__(data["source"]) magic_args = self.parse_cell_magic(self) if "-n" in magic_args: self.ignore = True magic_args.remove("-n") else:
def sign(self): notary = Notary() signature = notary.compute_signature(self.notebook) if not signature.startswith("sha256:"): signature = "sha256:" + signature self.notebook['metadata']['signature'] = signature