def execute_cells(kernel_name, cells, execute_kwargs):
"""Execute Jupyter cells in the specified kernel and return the notebook."""
notebook = blank_nb(kernel_name)
notebook.cells = cells
# Modifies 'notebook' in-place
try:
executenb(notebook, **execute_kwargs)
except Exception as e:
raise ExtensionError('Notebook execution failed', orig_exc=e)
return notebook
def get(self, path=None):
if path:
path = path.strip(self.base_url)
path += '.ipynb' # when used as a jupyter server extension, we don't use the extension
# if the handler got a notebook_path argument, always serve that
notebook_path = self.notebook_path or path
try:
notebook = nbformat.read(notebook_path, as_version=4)
except FileNotFoundError:
raise tornado.web.HTTPError(404)
# Ignore requested kernel name and make use of the one specified in the notebook.
kernel_name = notebook.metadata.get('kernelspec', {}).get('name', self.kernel_manager.default_kernel_name)
# Launch kernel and execute notebook.
kernel_id = yield tornado.gen.maybe_future(self.kernel_manager.start_kernel(kernel_name=kernel_name))
km = self.kernel_manager.get_kernel(kernel_id)
result = executenb(notebook, km=km)
# render notebook to html
resources = dict(kernel_id=kernel_id)
html, resources = HTMLExporter(template_file=str(TEMPLATE_ROOT / 'voila.tpl'), exclude_input=self.strip_sources,
exclude_output_prompt=self.strip_sources, exclude_input_prompt=self.strip_sources
).from_notebook_node(result, resources=resources)
# Compose reply
self.set_header('Content-Type', 'text/html')
self.write(html)
def execute(self, input_iterator, timeout=30):
"""This function is isolated from the cache, and is responsible for execution.
The method is only supplied with the staged record and input notebook bundle,
it then yield results for caching
"""
for stage_record, nb_bundle in input_iterator:
try:
uri = nb_bundle.uri
self.logger.info("Executing: {}".format(uri))
with tempfile.TemporaryDirectory() as tmpdirname:
try:
asset_files = copy_assets(stage_record, tmpdirname)
except Exception as err:
yield ExecutionError("Assets Retrieval Error", uri,
err)
continue
timer = Timer()
try:
with timer:
# execute notebook, transforming it in-place
# TODO does it need to wiped first?
if ("execution" in nb_bundle.nb.metadata
and "timeout"
in nb_bundle.nb.metadata.execution):
timeout = nb_bundle.nb.metadata.execution.timeout
executenb(nb_bundle.nb,
cwd=tmpdirname,
timeout=timeout)
except Exception:
exc_string = "".join(traceback.format_exc())
self.logger.error("Execution Failed: {}".format(uri))
yield create_bundle(
nb_bundle,
tmpdirname,
asset_files,
timer.last_split,
exc_string,
)
else:
self.logger.info("Execution Succeeded: {}".format(uri))
yield create_bundle(nb_bundle, tmpdirname, asset_files,
timer.last_split)
except Exception as err:
yield ExecutionError("Unexpected Error", uri, err)
def _block_and_receive_results(*return_values, save_output_notebook=None):
# add an extra cell to beginning of notebook to populate parameters
notebook_parameters = __notebookscripter_injected__ + [[hooks, {"return_values": return_values}]]
base64_parameters = obj_to_string_literal(notebook_parameters)
initialization_source = """from NotebookScripter import (rehydrate as __rehydrate__, dehydrate_return_values as __dehydrate_return_values__)
__rehydrate__({})""".format(base64_parameters)
initialization_cell = notebook_node_from_dict({
"cell_type": "code",
"execution_count": 0,
"metadata": {},
"outputs": [],
"source": initialization_source
})
finalization_source = """__dehydrate_return_values__(locals())"""
finalization_cell = notebook_node_from_dict({
"cell_type": "code",
"execution_count": 0,
"metadata": {},
"outputs": [],
"source": finalization_source})
notebook['cells'].insert(0, initialization_cell)
notebook['cells'].append(finalization_cell)
km = KernelManager()
# hack -- needed because the code within ExecutePreprocessor.start_kernel to start
# the kernel when km hasn't started a kernel already can't possibly work
km.start_kernel()
executed_notebook = executenb(notebook, timeout=None, km=km)
km.shutdown_kernel()
if save_output_notebook:
if isinstance(save_output_notebook, str):
with open(save_output_notebook, 'w') as f:
write_notebook(executed_notebook, f)
else:
write_notebook(executed_notebook, save_output_notebook)
encoded_return_values = eval(executed_notebook["cells"][-1]["outputs"][0]["data"]["text/plain"])
final_namespace = str_to_obj(encoded_return_values)
module_identity = "loaded_notebook_from_subprocess"
dynamic_module = types.ModuleType(module_identity)
dynamic_module.__file__ = path_to_notebook
# inject retrieved return values into the returned module namespace
dynamic_module.__dict__.update(final_namespace)
return dynamic_module
def get(self, path=None):
if path:
path += '.ipynb' # when used as a jupyter server extension, we don't use the extension
# if the handler got a notebook_path argument, always serve that
notebook_path = self.notebook_path or path
model = self.contents_manager.get(path=notebook_path)
if 'content' in model:
notebook = model['content']
else:
raise tornado.web.HTTPError(404, 'file not found')
# Fetch kernel name from the notebook metadata
kernel_name = notebook.metadata.get('kernelspec', {}).get(
'name', self.kernel_manager.default_kernel_name)
# Launch kernel and execute notebook
kernel_id = yield tornado.gen.maybe_future(
self.kernel_manager.start_kernel(kernel_name=kernel_name))
km = self.kernel_manager.get_kernel(kernel_id)
result = executenb(notebook, km=km)
# render notebook to html
resources = {'kernel_id': kernel_id, 'base_url': self.base_url}
exporter = HTMLExporter(template_file='voila.tpl',
template_path=self.template_path,
config=self.exporter_config)
if self.strip_sources:
exporter.exclude_input = True
exporter.exclude_output_prompt = True
exporter.exclude_input_prompt = True
# Filtering out empty cells.
def filter_empty_code_cells(cell):
return (
cell.cell_type != 'code' or # keep non-code cells
(cell.outputs and not exporter.exclude_output
) # keep cell if output not excluded and not empty
or
not exporter.exclude_input # keep cell if input not excluded
)
result.cells = list(filter(filter_empty_code_cells, result.cells))
html, resources = exporter.from_notebook_node(result,
resources=resources)
# Compose reply
self.set_header('Content-Type', 'text/html')
self.write(html)
def exe_check_nb_fname(nb_fname, wd=None):
""" Execute notebook, return error message or 'ok'
Parameters
----------
nb_fname : str
Filename of notebook to execute.
wd : None or str
Directory in which to execute notebook. None means use directory
containing `nb_fname`
Returns
-------
err_msg : str
Error message or 'ok' for no error.
"""
wd = op.dirname(nb_fname) if wd is None else wd
nb = jupytext.read(nb_fname)
try:
executenb(nb, cwd=wd, kernel_name=nb2kernel_name(nb))
except CellExecutionError as e:
return str(e)
return 'ok'
def get(self, path=None):
if path:
path += '.ipynb' # when used as a jupyter server extension, we don't use the extension
# if the handler got a notebook_path argument, always serve that
notebook_path = self.notebook_path or path
# generate a list of nbextensions that are enabled for the classical notebook
# a template can use that to load classical notebook extensions, but does not have to
notebook_config = self.config_manager.get('notebook')
# except for the widget extension itself, since voila has its own
load_extensions = notebook_config.get('load_extensions', {})
if "jupyter-js-widgets/extension" in load_extensions:
load_extensions["jupyter-js-widgets/extension"] = False
nbextensions = [
name for name, enabled in load_extensions.items() if enabled
]
else:
nbextensions = []
model = self.contents_manager.get(path=notebook_path)
if 'content' in model:
notebook = model['content']
else:
raise tornado.web.HTTPError(404, 'file not found')
# Fetch kernel name from the notebook metadata
kernel_name = notebook.metadata.get('kernelspec', {}).get(
'name', self.kernel_manager.default_kernel_name)
# Launch kernel and execute notebook
kernel_id = yield tornado.gen.maybe_future(
self.kernel_manager.start_kernel(kernel_name=kernel_name))
km = self.kernel_manager.get_kernel(kernel_id)
result = executenb(notebook, km=km)
# render notebook to html
resources = {
'kernel_id': kernel_id,
'base_url': self.base_url,
'nbextensions': nbextensions
}
exporter = HTMLExporter(template_file='voila.tpl',
template_path=self.nbconvert_template_paths,
config=self.exporter_config)
if self.strip_sources:
exporter.exclude_input = True
exporter.exclude_output_prompt = True
exporter.exclude_input_prompt = True
# Filtering out empty cells.
def filter_empty_code_cells(cell):
return (
cell.cell_type != 'code' or # keep non-code cells
(cell.outputs and not exporter.exclude_output
) # keep cell if output not excluded and not empty
or
not exporter.exclude_input # keep cell if input not excluded
)
result.cells = list(filter(filter_empty_code_cells, result.cells))
html, resources = exporter.from_notebook_node(result,
resources=resources)
# Compose reply
self.set_header('Content-Type', 'text/html')
self.write(html)
def run_notebook(nb_path, output_dir):
"""Run a notebook tests
executes the notebook and stores the output in a file
"""
import nbformat
from jupyter_client.kernelspec import KernelSpecManager
from nbconvert.preprocessors.execute import executenb
from datetime import datetime
log.info("Testing notebook " + str(nb_path))
with open(nb_path) as f:
nb = nbformat.read(f, as_version=4)
kernel_specs = KernelSpecManager().get_all_specs()
kernel_info = nb.metadata.get("kernelspec") or {}
kernel_name = kernel_info.get("name", "")
kernel_language = kernel_info.get("language") or ""
if kernel_name in kernel_specs:
log.info("Found kernel " + str(kernel_name))
elif kernel_language:
log.warning("No such kernel " + str(kernel_name) +
", falling back on kernel language=" +
str(kernel_language))
kernel_language = kernel_language.lower()
# no exact name match, re-implement js notebook fallback,
# using kernel language instead
# nbconvert does not implement this, but it should
for kernel_spec_name, kernel_info in kernel_specs.items():
if (kernel_info.get("spec",
{}).get("language",
"").lower() == kernel_language):
log.warning("Using kernel " + str(kernel_spec_name) +
" to provide language: " + str(kernel_language))
kernel_name = kernel_spec_name
break
else:
log.warning("Found no matching kernel for name=" +
str(kernel_name) + ", language=" +
str(kernel_language))
summary_specs = [
"name=" + str(name) + ", language=" +
str(info['spec'].get('language'))
for name, info in kernel_specs.items()
]
log.warning("Found kernel specs: " + '; '.join(summary_specs))
start_time = datetime.now()
exported = executenb(nb,
cwd=os.path.dirname(nb_path),
kernel_name=kernel_name,
timeout=600)
execution_time = (datetime.now() - start_time).seconds
log.info("Execution time is " + str(execution_time))
rel_path = os.path.relpath(nb_path, os.getcwd())
dest_path = os.path.join(output_dir, "notebooks", rel_path)
log.info("Saving exported notebook to " + str(dest_path))
try:
os.makedirs(os.path.dirname(dest_path))
except FileExistsError:
pass
with open(dest_path, "w") as f:
nbformat.write(exported, f)
from nbconvert.preprocessors.execute import executenb
import nbformat
import logging
logging.basicConfig(level=logging.DEBUG)
with open('Test.ipynb') as f:
nb = nbformat.read(f, 4)
executed = executenb(nb)
with open('Test_Executed.ipynb', 'w') as f:
nbformat.write(executed, f)
