def clear_notebook(path):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE="OPENADC",
PLATFORM="CWLITEARM")
new_nb = replace_definitions(nb, params, execute=False)
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
# remove release/autograded/feedback
if os.path.exists(os.path.join(root, "user_guide", "release")):
shutil.rmtree(os.path.join(root, "user_guide", "release"))
if os.path.exists(os.path.join(root, "user_guide", "autograded")):
shutil.rmtree(os.path.join(root, "user_guide", "autograded"))
if os.path.exists(os.path.join(root, "user_guide", "feedback")):
shutil.rmtree(os.path.join(root, "user_guide", "feedback"))
if os.path.exists(os.path.join(root, "user_guide", "downloaded", "ps1", "extracted")):
shutil.rmtree(os.path.join(root, "user_guide", "downloaded", "ps1", "extracted"))
print("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
km = self.kernel_manager.get_kernel(kernel_id)
nb, resources = ClearOutputPreprocessor().preprocess(
nb, {'metadata': {
'path': self.cwd
}})
ep = VoilaExecutePreprocessor(config=self.traitlet_config)
with ep.setup_preprocessor(nb, resources, km=km):
for cell_idx, cell in enumerate(nb.cells):
res = ep.preprocess_cell(cell,
resources,
cell_idx,
store_history=False)
yield res[0]
def clearNbOutput():
cwd = os.getcwd()
# walk the test directory and find all notebooks
for dirname, dirnames, filenames in os.walk(nbdir):
if dirname != ".ipynb_checkpoints":
for filename in filenames:
if (filename.endswith(".ipynb")
and not filename.endswith("-checkpoint.ipynb")):
with open(os.path.join(dirname, filename)) as f:
print("Clearing output from {}".format(filename))
nb = nbformat.read(f, as_version=4)
ep = ClearOutputPreprocessor()
ep.preprocess(nb, {
"kernel_name":
"python{}".format(sys.version_info[0])
})
print(" ... done\n")
return True
def unexecute(self):
"""
Unexecutes the notebook i.e. removes all output cells
"""
_logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}")
if not self.executed_nb_path.exists():
_logger.warning(
f"{self.executed_nb_path} not found, nothing to clean")
return
with open(self.executed_nb_path) as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
if nb['metadata'].get('docs_executed', None):
nb['metadata'].pop('docs_executed')
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w') as f:
nbformat.write(nb, f)
async def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
nb, resources = ClearOutputPreprocessor().preprocess(
nb, {'metadata': {
'path': self.cwd
}})
stop_execution = False
for cell_idx, cell in enumerate(nb.cells):
if stop_execution:
break
try:
res = await self.executor.execute_cell(cell,
None,
cell_idx,
store_history=False)
except TimeoutError:
res = cell
stop_execution = True
yield res
def clear_notebook(path):
"""Clears output from a notebook
Parameters
----------
path: str (optional)
path to a Jupyter Notebook
Returns
-------
None
"""
with open(_get_path(path), encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
pp = ClearOutputPreprocessor()
pp.preprocess(nb, {})
with open(path, "w", encoding="utf-8") as f:
nbformat.write(nb, f)
def clean():
_delete_generated_files()
config = Config()
config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()]
exporter = NotebookExporter(config=config)
for notebook in get_files("ipynb"):
with open(notebook, encoding="utf8") as nb_file:
nb = nbformat.read(nb_file, as_version=4)
if not nb.cells[-1]["source"]:
nb.cells.pop()
format_code_cells(nb)
for cell_id, cell in enumerate(nb.cells):
cell["id"] = f"{notebook.stem}-{cell_id}"
ipynb, _ = exporter.from_notebook_node(nb)
with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable:
writable.write(ipynb)
def nb_exec(fname, name):
ver = make_version(name)
dirpath = os.path.join(data_dir, name, ver)
execpath = os.path.join(dirpath, 'src')
src_nb = os.path.join(execpath, 'src.ipynb')
ensure_dir(dirpath)
ensure_dir(execpath)
logger.info(f'Copying {fname} to {src_nb}')
shutil.copy2(fname, src_nb)
with open(src_nb) as f:
nb = nbformat.read(f, as_version=4)
co = ClearOutputPreprocessor()
nb, _ = co.preprocess(nb, {})
ep = ExecutePreprocessor(timeout=-1, kernel_name='python3')
try:
ep.preprocess(nb, {'metadata': {'path': execpath}})
except CellExecutionError as e:
print(e)
with open(os.path.join(dirpath, 'error.log'), 'w') as f:
f.write(str(e))
print('Stack trace saved')
with open(os.path.join(dirpath, 'output.ipynb'), 'w',
encoding='utf-8') as f:
nbformat.write(nb, f)
exp = HTMLExporter()
body, res = exp.from_notebook_node(nb)
with open(os.path.join(dirpath, 'output.html'), 'w',
encoding='utf-8') as f:
f.write(body)
async def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
nb, resources = ClearOutputPreprocessor().preprocess(
nb, {'metadata': {
'path': self.cwd
}})
for cell_idx, input_cell in enumerate(nb.cells):
try:
task = asyncio.ensure_future(
self.executor.execute_cell(input_cell,
None,
cell_idx,
store_history=False))
while True:
done, pending = await asyncio.wait(
{task},
timeout=self.voila_configuration.
http_keep_alive_timeout)
if pending:
# If not done within the timeout, we send a heartbeat
# this is fundamentally to avoid browser/proxy read-timeouts, but
# can be used in a template to give feedback to a user
self.write("<script>voila_heartbeat()</script>\n")
self.flush()
continue
output_cell = await task
break
except TimeoutError:
output_cell = input_cell
break
except CellExecutionError:
if self.executor.should_strip_error():
strip_code_cell_warnings(input_cell)
self.executor.strip_code_cell_errors(input_cell)
output_cell = input_cell
break
except Exception as e:
self.log.exception('Error at server while executing cell: %r',
input_cell)
output_cell = nbformat.v4.new_code_cell()
if self.executor.should_strip_error():
output_cell.outputs = [{
"output_type":
"stream",
"name":
"stderr",
"text":
"An exception occurred at the server (not the notebook). {}"
.format(self.executor.cell_error_instruction),
}]
else:
output_cell.outputs = [{
'output_type':
'error',
'ename':
type(e).__name__,
'evalue':
str(e),
'traceback':
traceback.format_exception(*sys.exc_info()),
}]
finally:
yield output_cell
def open_nb_and_strip_output(fname):
cop = ClearOutputPreprocessor()
with open(fname) as f:
nb = nbformat.read(f, as_version=4)
cop.preprocess(nb, dict())
return nb
def convert_notebook_to_assets(notebook_file_name, base_name, output_prefix):
# define and create output folder
output_folder = output_prefix + '/' + base_name
os.makedirs(output_folder, exist_ok=True)
# open file
print('Converting Notebook: ' + notebook_file_name + ' ...')
nb_file = open(notebook_file_name, 'r').read()
nb = nbformat.reads(nb_file, as_version=4)
# 1. clear output
print(" - clearing output")
ep = ClearOutputPreprocessor()
ep.preprocess(nb, {})
# 2. generate fresh charts by running the notebook
print(" - executing")
ep = ExecutePreprocessor(timeout=600,
kernel_name='python3',
allow_errors=False)
try:
ep.preprocess(nb, {'metadata': {'path': output_folder}})
except Exception as e:
print('ERROR: Execution of the notebook ' + notebook_file_name +
' stopped, likely for missing some py libs.')
print(
' Please check the output/exception and add those to the requirements.'
)
print(e)
exit(1)
# 3. export HTML
print(" - generating html")
cleaner_config = Config({
"HTMLExporter": {
"exclude_input": True,
"exclude_input_prompt": True,
"exclude_output_prompt": True,
"preprocessors":
['nbconvert.preprocessors.ExtractOutputPreprocessor']
},
})
local_templates = DictLoader({
'our-html.tpl': stand_alone_tpl,
'react-glue.tpl': react_glue_tpl,
})
exporter = HTMLExporter(config=cleaner_config,
extra_loaders=[local_templates])
exporter.template_file = 'our-html.tpl'
(html_body, html_resources) = exporter.from_notebook_node(nb)
notebook_convert_time = current_utc_time()
# save html output file, with local reference to the pictures
local_html = []
output_html_file_name = output_folder + '/' + "index.html"
print(" - saving html: " + output_html_file_name)
with open(output_html_file_name, 'wt') as the_file:
the_file.write(html_body)
local_html.append({
'notebook': base_name,
'notebook_html': output_html_file_name,
'convert_time': notebook_convert_time,
})
# save js file for react inclusion (local ref to the pictures)
# exporter.template_file = 'react-glue.tpl'
# (react_body, react_resources) = exporter.from_notebook_node(nb)
# output_react_file_name = output_folder + '/' + "index.js"
# print(" - saving react js: " + output_react_file_name)
# with open(output_react_file_name, 'wt') as the_file:
# the_file.write(react_body)
# save all the figures
local_figures = []
figures = html_resources['outputs']
figures_count = len(figures)
figure_index = 1
for figure_file in figures:
output_figure_file_name = output_folder + '/' + figure_file
print(" - saving png " + str(figure_index) + " of " +
str(figures_count) + ": " + output_figure_file_name)
if not figure_file.endswith('.png'):
print("WARNING: figure is not a PNG file")
continue
with open(output_figure_file_name, 'wb') as the_file:
the_file.write(figures[figure_file])
local_figures.append({
'figure': figure_file,
'file': output_figure_file_name,
'notebook': base_name,
'notebook_html': output_html_file_name,
'convert_time': notebook_convert_time,
})
# create an empty 'custom.css'
custom_css_file_name = output_folder + '/' + 'custom.css'
with open(custom_css_file_name, 'wt') as the_file:
the_file.write("")
# return a recap of all assets
return local_html, local_figures
import os
import glob
from nbformat import read, write, current_nbformat
from nbconvert.preprocessors import ClearOutputPreprocessor
notebooks = glob.glob("*/*.ipynb")
for nbpath in notebooks:
print(nbpath)
with open(nbpath, "r") as fh:
nb = read(fh, current_nbformat)
for cell in nb.cells:
cell.metadata = {}
nb, _ = ClearOutputPreprocessor().preprocess(nb, {})
nb.metadata = {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
}
}
with open(nbpath, "w") as fh:
write(nb, fh, current_nbformat)
def test_func(self):
cwd = os.getcwd()
passing = True
print("\n---------------------"
" Testing {0}.ipynb "
"---------------------".format(nbname))
if (nbname in self.ignore) or (nbname in self.py2_ignore
and sys.version_info[0] == 2):
print(" Skipping {}".format(nbname))
return
run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1])
os.chdir(run_path)
clear_output = ClearOutputPreprocessor()
with open(nbpath) as nbfile:
notebook = nbformat.read(nbfile, as_version=4)
clear_output.preprocess(notebook, {})
execute = ExecutePreprocessor(
timeout=timeout,
kernel_name="python{}".format(sys.version_info[0]),
allow_errors=True,
)
out = execute.preprocess(notebook, {})
os.chdir(cwd)
for cell in out[0]["cells"]:
if "outputs" in cell.keys():
for output in cell["outputs"]:
if output["output_type"] == "error":
passing = False
err_msg = []
for traceback in output["traceback"]:
err_msg += ["{}".format(traceback)]
err_msg = "\n".join(err_msg)
msg = """
\n ... {} FAILED \n
{} in cell [{}] \n-----------\n{}\n-----------\n
""".format(
nbname,
output["ename"],
cell["execution_count"],
cell["source"],
)
traceback = """
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(err_msg)
print(u"{}".format(msg + traceback))
assert passing, msg
print(" ... {0} Passed \n".format(nbname))
def test_func(self):
cwd = os.getcwd()
passing = True
print("\n---------------------"
" Testing {0}.ipynb "
"---------------------".format(nbname))
if ((nbname in self.ignore)
or (nbname in self.py2_ignore and sys.version_info[0] == 2)):
print(" Skipping {}".format(nbname))
return
run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1])
os.chdir(run_path)
ep = ClearOutputPreprocessor(
resources={'metadata': {
'path': run_path
}})
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=timeout,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True,
resources={'metadata': {
'path': run_path
}})
out = ex.preprocess(nb, {})
os.chdir(cwd)
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error':
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n ... {} FAILED \n
{} in cell [{}] \n-----------\n{}\n-----------\n
""".format(
nbname,
output['ename'],
cell['execution_count'],
cell['source'],
)
traceback = """
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(err_msg)
print(u"{}".format(msg + traceback))
assert passing, msg
print(" ... {0} Passed \n".format(nbname))
def execute(self, force=False):
"""
Executes the specified notebook file, and writes the executed notebook to a
new file.
Parameters
----------
force : bool, optional
To force rerun notebook even if it has already been executed
Returns
-------
executed_nb_path : str, ``None``
The path to the executed notebook path, or ``None`` if ``write=False``.
status: bool
Whether the notebook executed without errors or not, 0 = ran without error, 1 = error
"""
with open(self.nb_path, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
is_executed = nb['metadata'].get('docs_executed')
if is_executed == 'executed' and not force:
_logger.warning(
f"Notebook {self.nb} in {self.nb_dir} already executed, skipping,"
f"to force execute, parse argument -f")
status = 0
else:
# Execute the notebook
_logger.info(f"Executing notebook {self.nb} in {self.nb_dir}")
t0 = time.time()
clear_executor = ClearOutputPreprocessor()
executor = ExecutePreprocessor(**self.execute_kwargs)
# First clean up the notebook and remove any cells that have been run
clear_executor.preprocess(nb, {})
try:
executor.preprocess(nb, {'metadata': {'path': self.nb_dir}})
execute_dict = {'docs_executed': 'executed'}
nb['metadata'].update(execute_dict)
status = 0
except CellExecutionError as err:
execute_dict = {'docs_executed': 'errored'}
nb['metadata'].update(execute_dict)
_logger.error(f"Error executing notebook {self.nb}")
_logger.error(err)
status = 1
_logger.info(f"Finished running notebook ({time.time() - t0})")
_logger.info(
f"Writing executed notebook to {self.executed_nb_path}")
# Makes sure original notebook isn't left blank in case of error during writing
if self.overwrite:
with open(self.temp_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
shutil.copyfile(self.temp_nb_path, self.executed_nb_path)
os.remove(self.temp_nb_path)
else:
with open(self.executed_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
return self.executed_nb_path, status
def clear_output(self, nb):
from nbconvert.preprocessors import ClearOutputPreprocessor
return ClearOutputPreprocessor().preprocess(nb, {})
def run_notebook(self, name, event=None, timeout=None):
""" run a given notebook immediately.
Args:
name (str): the name of the jobfile
event (str): an event name
timeout (int): timeout in seconds
Returns:
Metadata of results
See Also:
* nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html
"""
notebook = self.get(name)
meta_job = self.metadata(name)
ts = datetime.datetime.now()
# execute kwargs
# -- see ExecuteProcessor class
# -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html
ep_kwargs = {
# avoid timeouts to stop kernel
'timeout': timeout,
# avoid kernel at exit functions
# -- this stops ipykernel AttributeError 'send_multipart'
'shutdown_kernel': 'immediate',
# set kernel name, blank is default
# -- e.g. python3, ir
# -- see https://stackoverflow.com/a/47053020/890242
'kernel_name': '',
}
# other interesting options
ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {}))
try:
resources = {
'metadata': {
'path': self.defaults.OMEGA_TMP,
}
}
if not meta_job.kind_meta.get('keep_output', False):
# https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html
cp = ClearOutputPreprocessor()
cp.preprocess(notebook, resources)
ep = ExecutePreprocessor(**ep_kwargs)
ep.preprocess(notebook, resources)
except Exception as e:
status = 'ERROR'
message = str(e)
else:
status = 'OK'
message = ''
finally:
del ep
# record results
meta_results = self.put(notebook,
'results/{name}_{ts}'.format(**locals()))
meta_results.attributes['source_job'] = name
meta_results.save()
job_results = meta_job.attributes.get('job_results', [])
job_results.append(meta_results.name)
meta_job.attributes['job_results'] = job_results
# record final job status
job_runs = meta_job.attributes.get('job_runs', [])
runstate = {
'status': status,
'ts': ts,
'message': message,
'results': meta_results.name if status == 'OK' else None
}
job_runs.append(runstate)
meta_job.attributes['job_runs'] = job_runs
# set event run state if event was specified
if event:
attrs = meta_job.attributes
triggers = attrs['triggers'] = attrs.get('triggers', [])
scheduled = (trigger for trigger in triggers
if trigger['event-kind'] == 'scheduled')
for trigger in scheduled:
if event == trigger['event']:
trigger['status'] = status
trigger['ts'] = ts
meta_job.save()
return meta_results