def run_notebook(path, timeout=120):
"""
Execute a Jupyter Notebook and return any errors that it produces.
:param path: path to the .ipynb file to execute
:param int timeout: number of seconds to let the notebook run before we throw an exception by default.
:return: a tuple (nb, errors) containing the parsed Notebook object and a list of errors, respectively
"""
# We'll use a NotebookExporter from the nbconvert package to load the notebook and re-export it to a temporary file.
# First we want to configure it to execute the notebook before writing it:
c = Config()
c.NotebookExporter.preprocessors = ['nbconvert.preprocessors.ExecutePreprocessor']
c.ExecutePreprocessor.timeout = timeout
c.ExecutePreprocessor.kernel_name = 'LEAP_venv' # We assume a kernel named "LEAP_venv" that lives in our venv
exp = NotebookExporter(config=c)
# Load the notebook
with open(path, 'r') as nb_file:
body, resources = exp.from_file(nb_file)
# Parse the notebook string into a notebook object
nb = nbformat.reads(body, nbformat.current_nbformat)
errors = [output for cell in nb.cells if "outputs" in cell
for output in cell["outputs"]
if output.output_type == "error"]
return nb, errors
def _clear_notebook(self, node): # pragma: no cover
if NotebookExporter is not None:
exporter = NotebookExporter()
exporter.register_preprocessor(ClearOutputPreprocessor(enabled=True))
cleared,_ = exporter.from_notebook_node(node)
else:
stripped_node = v3_strip_output(node)
cleared = current.writes(stripped_node, 'ipynb')
return cleared
def evaluate_notebook(nb_path,
dest_path=None,
skip_exceptions=False,
skip_execute=None,
timeout=300,
ipython_startup=None,
patterns_to_take_with_me=None):
if patterns_to_take_with_me is None:
patterns_to_take_with_me = []
notebook = nbformat.read(nb_path, as_version=4)
kwargs = dict(timeout=timeout,
kernel_name='python%s' % sys.version_info[0],
allow_errors=skip_exceptions)
cwd = os.getcwd()
filedir, filename = os.path.split(nb_path)
os.chdir(filedir)
not_nb_runner = ExecutePreprocessor1000(**kwargs)
if ipython_startup is not None:
not_nb_runner._ipython_startup = ipython_startup
if not os.path.isfile(dest_path):
print('INFO: Writing evaluated notebook to {dest_path!s}'.format(
dest_path=os.path.abspath(dest_path)))
try:
if not skip_execute:
not_nb_runner.preprocess(notebook, {})
except CellExecutionError as e:
print('')
print(e)
os.chdir(cwd)
if skip_execute:
nbformat.write(notebook, open(dest_path, 'w'))
else:
ne = NotebookExporter()
newnb, _ = ne.from_notebook_node(notebook)
with open(dest_path, 'w') as f:
f.write(newnb)
for pattern in patterns_to_take_with_me:
for f in glob.glob(
os.path.join(os.path.dirname(nb_path), pattern)):
print("mv %s %s" % (f, os.path.dirname(dest_path)))
shutil.move(f, os.path.dirname(dest_path))
else:
print(
'INFO: Skipping existing evaluated notebook {dest_path!s}'.format(
dest_path=os.path.abspath(dest_path)))
def render_track(track, track_cfg):
meta = utils.get_track_meta(track, track_cfg)
cfg = Config()
cfg.Exporter.preprocessors = ['nb_utils.lesson_preprocessor.LearnLessonPreprocessor']
exporter = NotebookExporter(config=cfg)
resources = {'track_meta': meta, 'track_cfg': track_cfg}
outdir = os.path.join(track, track_cfg['tag'], 'rendered')
os.makedirs(outdir, exist_ok=True)
for nb_meta in meta.notebooks:
in_path = os.path.join(track, 'raw', nb_meta.filename)
resources['lesson'] = nb_meta.lesson
resources['nb_meta'] = nb_meta
if CLEAN:
clean(in_path)
nb, _ = exporter.from_filename(in_path, resources)
out_path = os.path.join(outdir, nb_meta.filename)
with open(out_path, 'w') as f:
f.write(nb)
def clean():
_delete_generated_files()
config = Config()
config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()]
exporter = NotebookExporter(config=config)
for notebook in get_files("ipynb"):
with open(notebook, encoding="utf8") as nb_file:
nb = nbformat.read(nb_file, as_version=4)
if not nb.cells[-1]["source"]:
nb.cells.pop()
format_code_cells(nb)
for cell_id, cell in enumerate(nb.cells):
cell["id"] = f"{notebook.stem}-{cell_id}"
ipynb, _ = exporter.from_notebook_node(nb)
with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable:
writable.write(ipynb)
def render_track(track, nb_path_whitelist=None):
meta = utils.get_track_meta(track)
track_cfg = utils.get_track_config(track)
cfg = Config()
cfg.Exporter.preprocessors = [
'lesson_preprocessor.LearnLessonPreprocessor'
]
exporter = NotebookExporter(config=cfg)
resources = {'track_meta': meta, 'track_cfg': track_cfg}
for nb_meta in meta.notebooks:
in_path = os.path.join(track, 'raw', nb_meta.filename)
if nb_path_whitelist and in_path not in nb_path_whitelist:
continue
resources['lesson'] = nb_meta.lesson
resources['nb_meta'] = nb_meta
if CLEAN:
clean(in_path)
nb, _ = exporter.from_filename(in_path, resources)
out_path = os.path.join(track, 'rendered', nb_meta.filename)
with open(out_path, 'w') as f:
f.write(nb)
def main():
nb = nbformat.read('04-merge-pivot-answers.ipynb',
as_version=nbformat.NO_CONVERT)
# In[5]:
exporter = NotebookExporter()
body, resources = exporter.from_notebook_node(nb)
# In[6]:
c = Config()
c.NotebookExporter.preprocessors = [ExercisePreprocessor]
exercise_exporter = NotebookExporter(config=c)
# exercise_exporter.preprocessors
# In[7]:
body2, res2 = exercise_exporter.from_notebook_node(nb)
# In[8]:
nb2 = nbformat.reads(body2, 4)
'CachedOutputPreprocessor': {
'enabled':
True,
'cache_directory':
'.nb_output_cache',
'req_files': [
'code/init_mooc_nb.py', 'code/edx_components.py',
'code/pfaffian.py', 'code/functions.py'
],
'warnings':
'ignore',
'timeout':
300
}
})
cachedoutput = NotebookExporter(cfg)
with open('scripts/release_dates') as f:
release_dates = eval(f.read())
def date_to_edx(date, add_days=0):
tmp = strptime(date, '%d %b %Y')
date = datetime.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, 10)
date = date + datetime.timedelta(days=add_days)
date = date.strftime('%Y-%m-%dT%H:%M:%SZ')
return date
def parse_syllabus(syllabus_file, content_folder=''):
preprocessor_list.append(FormatCodeCellPreprocessor)
if execute_code:
preprocessor_list.append(preprocessors.ExecutePreprocessor)
# Create the exporters with preprocessing
c = Config()
c.NotebookExporter.preprocessors = preprocessor_list
c.HTMLExporter.preprocessors = preprocessor_list
if execute_code:
c.ExecutePreprocessor.timeout = cell_timeout
if execute_code != "default":
c.ExecutePreprocessor.kernel_name = execute_code
nb_exporter = NotebookExporter(c)
html_exporter = HTMLExporter(c)
# html_exporter.template_file = 'basic'
# Find all Jupyter notebook files in the specified directory
all_files = []
for p in args.locations:
path = Path(p)
if path.is_dir():
all_files.extend(path.glob("**/*.ipynb"))
elif path.is_file():
all_files.append(path)
else:
raise ValueError(
f"Specified location not '{path}'a file or directory.")
preprocessor_list.append(ClearWarningsPreprocessor)
if coalesce_streams:
preprocessor_list.append(preprocessors.coalesce_streams)
# Create the exporters with preprocessing
c = Config()
c.NotebookExporter.preprocessors = preprocessor_list
c.HTMLExporter.preprocessors = preprocessor_list
if execute_code:
c.ExecutePreprocessor.timeout = cell_timeout
if execute_code != "default":
c.ExecutePreprocessor.kernel_name = execute_code
nb_exporter = NotebookExporter(c)
html_exporter = HTMLExporter(c)
# html_exporter.template_file = 'basic'
# Find all Jupyter notebook files in the specified directory
all_files = []
for p in args.locations:
path = Path(p)
if path.is_dir():
all_files.extend(path.glob("**/*.ipynb"))
elif path.is_file():
all_files.append(path)
else:
raise ValueError(
f"Specified location not '{path}'a file or directory.")
c = Config()
c.NotebookExporter.preprocessors = [
"nbconvert.preprocessors.RegexRemovePreprocessor",
"nbconvert.preprocessors.ExecutePreprocessor",
]
# Remove any cells containing lines with the phrase "Table of Contents" or
# consist solely of whitespace
c.RegexRemovePreprocessor.enabled = True
c.RegexRemovePreprocessor.patterns = [".*Table of Contents.*", "\\s*\\Z"]
# Execute each notebook in a python3 kernel to make sure the content is there
c.ExecutePreprocessor.enabled = True
c.ExecutePreprocessor.kernel = "python3"
exporter = NotebookExporter(config=c)
for nb in os.listdir(os.path.join(ROOT, "examples")):
# Skip any non-notebooks
if nb.endswith("ipynb") and "no-doc" not in nb:
nb_in = os.path.join(ROOT, "examples", nb)
nb_out = os.path.join(ROOT, "docs/source/read-only-examples", nb)
# Skip any notebooks which already exist
if not os.path.exists(nb_out):
with open(nb_out, "w") as out_stream:
converted = exporter.from_filename(nb_in)[0]
out_stream.write(converted)
import skcosmo # noqa
def evaluate_notebook(nb_path,
dest_path=None,
skip_exceptions=False,
substring=None,
end=None,
skip_execute=None,
skip_output=None,
offset=0,
timeout=300,
ipython_startup=None,
patterns_to_take_with_me=None):
if patterns_to_take_with_me is None:
patterns_to_take_with_me = []
notebook = nbformat.read(nb_path, as_version=4)
kwargs = dict(timeout=timeout,
kernel_name='python%s' % sys.version_info[0],
allow_errors=skip_exceptions)
cwd = os.getcwd()
filedir, filename = os.path.split(nb_path)
os.chdir(filedir)
#profile_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'profile_docs')
#print("Ignoring %s...hope that's ok"%profile_dir)
not_nb_runner = ExecutePreprocessor1000(**kwargs)
if ipython_startup is not None:
not_nb_runner._ipython_startup = ipython_startup
if not os.path.isfile(dest_path):
# TODO but this isn't true, is it? it's running the originl nb
print('INFO: Writing evaluated notebook to {dest_path!s}'.format(
dest_path=os.path.abspath(dest_path)))
try:
if not skip_execute:
not_nb_runner.preprocess(notebook, {})
except CellExecutionError as e:
print('')
print(e)
os.chdir(cwd)
if skip_execute:
nbformat.write(notebook, open(dest_path, 'w'))
else:
ne = NotebookExporter()
newnb, _ = ne.from_notebook_node(notebook)
with open(dest_path, 'w') as f:
f.write(newnb)
for pattern in patterns_to_take_with_me:
for f in glob.glob(
os.path.join(os.path.dirname(nb_path), pattern)):
print("mv %s %s" % (f, os.path.dirname(dest_path)))
shutil.move(f, os.path.dirname(dest_path))
else:
print(
'INFO: Skipping existing evaluated notebook {dest_path!s}'.format(
dest_path=os.path.abspath(dest_path)))
preprocessors = [] if substring is None and not offset else [
NotebookSlice(substring, end, offset)
]
preprocessors = (
preprocessors +
[SkipOutput(skip_output)]) if skip_output else preprocessors
ret = nb_to_html(dest_path, preprocessors=preprocessors)
return ret
def to_notebook(self):
return NotebookExporter().from_notebook_node(self._content)[0]
class CleanTagReplacer(Preprocessor):
"""Replace cells tagged 'clean' with a dummy cell"""
def preprocess(
self, nb: "NotebookNode",
resources: Dict[Any,
Any]) -> Tuple["NotebookNode", Dict[Any, Any]]:
"""Preprocess the entire notebook."""
for i, cell in enumerate(nb.cells):
if "tags" in cell.metadata and "clean" in cell.metadata["tags"]:
nb.cells[i] = new_code_cell(
source="# Write Python source code in this cell")
return nb, resources
nb_exp = NotebookExporter(preprocessors=[CleanTagReplacer])
if __name__ == "__main__":
HERE = Path(__file__).parent
output_directory = HERE.joinpath("output")
for notebook in HERE.glob("*.ipynb"):
output_notebook = notebook.with_stem(notebook.stem + "-clean").name
nb = nbformat.read(notebook, as_version=4)
nb_new = upgrade(nb)
if nb_new is not None:
nb = nb_new
if "celltoolbar" in nb.metadata:
del nb.metadata["celltoolbar"]
clean_nb, _ = nb_exp.from_notebook_node(nb)
