def read_nb(filename, profile_dir=None, encoding="utf8", working_dir=None,
comment="", fLOG=noLOG, code_init=None,
kernel_name="python", log_level="30", extended_args=None,
kernel=False, replacements=None):
"""
reads a notebook and return a @see cl NotebookRunner object
@param filename notebook filename (or stream)
@param profile_dir profile directory
@param encoding encoding for the notebooks
@param working_dir working directory
@param comment additional information added to error message
@param code_init to initialize the notebook with a python code as if it was a cell
@param fLOG logging function
@param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
@param kernel_name kernel name, it can be None
@param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
see :ref:`l-ipython_notebook_args` for a full list
@param kernel *kernel* is True by default, the notebook can be run, if False,
the notebook can be read but not run
@param replacements replacements to make in every cell before running it,
dictionary ``{ string: string }``
@return @see cl NotebookRunner
.. versionchanged:: 1.3
Parameter *kernel* was added.
.. versionchanged:: 1.5
Parameters from NotebookRunner were added.
"""
if isinstance(filename, str # unicode#
):
with open(filename, "r", encoding=encoding) as payload:
nb = reads(payload.read())
nb_runner = NotebookRunner(
nb, profile_dir=profile_dir, theNotebook=os.path.abspath(filename),
kernel=kernel, working_dir=working_dir,
comment=comment, fLOG=fLOG, code_init=code_init,
kernel_name="python", log_level="30", extended_args=None,
filename=filename, replacements=replacements)
return nb_runner
else:
nb = reads(filename.read())
nb_runner = NotebookRunner(nb, kernel=kernel,
profile_dir=profile_dir, working_dir=working_dir,
comment=comment, fLOG=fLOG, code_init=code_init,
kernel_name="python", log_level="30", extended_args=None,
filename=filename, replacements=replacements)
return nb_runner
def _test_notebook(self, notebook, test):
with open(notebook) as f:
nb = convert(reads(f.read()), self.NBFORMAT_VERSION)
_, kernel = utils.start_new_kernel()
for i, c in enumerate([c for c in nb.cells if c.cell_type == 'code']):
self._test_notebook_cell(self.sanitize_cell(c), i, kernel, test)
def _test_notebook(self, notebook, test):
with open(notebook) as f:
nb = convert(reads(f.read()), self.NBFORMAT_VERSION)
_, kernel = utils.start_new_kernel()
for i, c in enumerate([c for c in nb.cells if c.cell_type == 'code']):
self._test_notebook_cell(self.sanitize_cell(c), i, kernel, test)
def upgrade_notebook(filename, encoding="utf8"):
"""
converts a notebook from version 2 to 3
@param filename filename
@param encoding encoding
@return modification?
.. versionadded:: 1.0
"""
with open(filename, "r", encoding=encoding) as payload:
content = payload.read()
nb = reads(content)
if not hasattr(nb, "nbformat") or nb.nbformat >= 4:
return False
upgrade(nb, from_version=nb.nbformat)
s = writes(nb)
if isinstance(s, bytes):
s = s.decode('utf8')
if s == content:
return False
else:
with open(filename, "w", encoding=encoding) as f:
f.write(s)
return True
def _get_notebook_node(self): # pragma: no cover
"Load captured notebook node"
size = len(self._notebook_data)
if size == 0:
raise Exception("Captured buffer size for notebook node is zero.")
node = reader.reads(self._notebook_data)
self.nbversion = reader.get_version(node)
return node
def _get_notebook_node(self): # pragma: no cover
"Load captured notebook node"
size = len(self._notebook_data)
if size == 0:
raise Exception("Captured buffer size for notebook node is zero.")
node = reader.reads(self._notebook_data)
self.nbversion = reader.get_version(node)
return node
def read_notebook(fn):
from nbformat.converter import convert
from nbformat.reader import reads
IPYTHON_VERSION = 4
NBFORMAT_VERSION = 4
with open(fn, 'r') as f:
text = f.read()
data = reads(text)
return text, convert(data, NBFORMAT_VERSION)
def _get_notebook_node(self): # pragma: no cover
"Load captured notebook node"
self._notebook_data.seek(0, os.SEEK_END)
size = self._notebook_data.tell()
if size == 0:
raise Exception("Captured buffer size for notebook node is zero.")
self._notebook_data.seek(0)
node = reader.reads(self._notebook_data.read())
self.nbversion = reader.get_version(node)
self._notebook_data.close()
return node
def _get_notebook_node(self): # pragma: no cover
"Load captured notebook node"
self._notebook_data.seek(0, os.SEEK_END)
size = self._notebook_data.tell()
if size == 0:
raise Exception("Captured buffer size for notebook node is zero.")
self._notebook_data.seek(0)
node = reader.reads(self._notebook_data.read())
self.nbversion = reader.get_version(node)
self._notebook_data.close()
return node
def copy(self):
"""
Copies the notebook (just the content).
@return instance of @see cl NotebookRunner
"""
st = StringIO()
self.to_json(st)
args = self.init_args.copy()
for name in ["theNotebook", "filename"]:
if name in args:
del args[name]
nb = reads(st.getvalue())
return NotebookRunner(nb, **args)
def upgrade_notebook(filename, encoding="utf-8"):
"""
Converts a notebook from version 2 to latest.
@param filename filename
@param encoding encoding
@return modification?
.. versionadded:: 1.0
"""
with open(filename, "r", encoding=encoding) as payload:
content = payload.read()
try:
nb = reads(content)
except NotJSONError as e:
if len(content) > 10:
lc = list(content[:10])
else:
lc = list(content)
raise ValueError(
"Unable to read content type '{0}' in '{2}' ---- {1}".format(
type(content), lc, filename)) from e
if not hasattr(nb, "nbformat") or nb.nbformat >= 4:
return False
try:
upgrade(nb, from_version=nb.nbformat)
except ValueError as e:
raise ValueError("Unable to convert '{0}'.".format(filename)) from e
s = writes(nb)
if isinstance(s, bytes):
s = s.decode('utf8')
if s == content:
return False
else:
with open(filename, "w", encoding=encoding) as f:
f.write(s)
return True
tv = IPyTestConsole()
if args.strict:
tv.default_results['diff'] = False
if args.no_timeout:
tv.default_results['timeout'] = False
tv.writeln('testing ipython notebook : "%s"' % ipynb)
tv.fold_open('ipynb')
timeout_rerun = args.rerun
fail_restart = args.restart
with open(ipynb) as f:
nb = reads(f.read())
# Convert all notebooks to the format IPython 4.0.0 uses to
# simplify comparison
nb = nb_convert(nb, 4)
notebook_restart = True
notebook_run_count = 0
while (notebook_restart):
notebook_restart = False
notebook_run_count += 1
tv.reset()
tv.write("starting kernel ... ")
with IPyKernel() as ipy:
ipy.default_timeout = args.timeout
def _readnb(self, filename):
with open(filename) as f:
return reads(f.read())
def _readnb(self, filename):
with open(filename) as f:
return reads(f.read())
def read_nb(filename,
profile_dir=None,
encoding="utf8",
working_dir=None,
comment="",
fLOG=noLOG,
code_init=None,
kernel_name="python",
log_level="30",
extended_args=None,
kernel=False,
replacements=None):
"""
Reads a notebook and return a @see cl NotebookRunner object.
@param filename notebook filename (or stream)
@param profile_dir profile directory
@param encoding encoding for the notebooks
@param working_dir working directory
@param comment additional information added to error message
@param code_init to initialize the notebook with a python code as if it was a cell
@param fLOG logging function
@param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
@param kernel_name kernel name, it can be None
@param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
see :ref:`l-ipython_notebook_args` for a full list
@param kernel *kernel* is True by default, the notebook can be run, if False,
the notebook can be read but not run
@param replacements replacements to make in every cell before running it,
dictionary ``{ string: string }``
@return @see cl NotebookRunner
.. versionchanged:: 1.3
Parameter *kernel* was added.
.. versionchanged:: 1.5
Parameters from NotebookRunner were added.
"""
if isinstance(
filename,
str # unicode#
):
with open(filename, "r", encoding=encoding) as payload:
nb = reads(payload.read())
nb_runner = NotebookRunner(nb,
profile_dir=profile_dir,
theNotebook=os.path.abspath(filename),
kernel=kernel,
working_dir=working_dir,
comment=comment,
fLOG=fLOG,
code_init=code_init,
kernel_name="python",
log_level="30",
extended_args=None,
filename=filename,
replacements=replacements)
return nb_runner
else:
nb = reads(filename.read())
nb_runner = NotebookRunner(nb,
kernel=kernel,
profile_dir=profile_dir,
working_dir=working_dir,
comment=comment,
fLOG=fLOG,
code_init=code_init,
kernel_name="python",
log_level="30",
extended_args=None,
filename=filename,
replacements=replacements)
return nb_runner
def run_notebook(filename,
profile_dir=None,
working_dir=None,
skip_exceptions=False,
outfilename=None,
encoding="utf8",
additional_path=None,
valid=None,
clean_function=None,
code_init=None,
fLOG=noLOG,
kernel_name="python",
log_level="30",
extended_args=None,
cache_urls=None,
replacements=None,
detailed_log=None,
startup_timeout=300):
"""
Runs a notebook end to end,
it is inspired from module `runipy <https://github.com/paulgb/runipy/>`_.
@param filename notebook filename
@param profile_dir profile directory
@param working_dir working directory
@param skip_exceptions skip exceptions
@param outfilename if not None, saves the output in this notebook
@param encoding encoding for the notebooks
@param additional_path additional paths for import
@param valid if not None, valid is a function which returns whether
or not the cell should be executed or not, if the function
returns None, the execution of the notebooks and skip the execution
of the other cells
@param clean_function function which cleans a cell's code before executing it (None for None)
@param code_init code to run before the execution of the notebook as if it was a cell
@param fLOG logging function
@param kernel_name kernel name, it can be None
@param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
@param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
see :ref:`l-ipython_notebook_args` for a full list
@param cache_urls list of urls to cache
@param replacements list of additional replacements, list of tuple
@param detailed_log a second function to log more information when executing the notebook,
this should be a function with the same signature as ``print`` or None
@param startup_timeout wait for this long for the kernel to be ready,
see `wait_for_ready
<https://github.com/jupyter/jupyter_client/blob/master/jupyter_client/blocking/client.py#L84>`_
@return tuple (statistics, output)
@warning The function calls `basicConfig
<https://docs.python.org/3/library/logging.html#logging.basicConfig>`_.
.. exref::
:title: Run a notebook end to end
::
from pyquickhelper.ipythonhelper import run_notebook
run_notebook("source.ipynb", working_dir="temp",
outfilename="modified.ipynb",
additional_path=["custom_path"] )
The function adds the local variable ``theNotebook`` with
the absolute file name of the notebook.
The execution of a notebook might fail because it relies on remote data
specified by url. The function downloads the data first and stores it in
folder *working_dir* (must not be None). The url string is replaced by
the absolute path to the file.
.. versionchanged:: 1.8
Parameters *detailed_log*, *startup_timeout* were added.
"""
cached_rep = _cache_url_to_file(cache_urls, working_dir, fLOG=fLOG)
if replacements is None:
replacements = cached_rep
elif cached_rep is not None:
cached_rep.update(replacements)
else:
cached_rep = replacements
with open(filename, "r", encoding=encoding) as payload:
try:
nbc = payload.read()
except UnicodeDecodeError as e: # pragma: no cover
raise NotebookException(
"(2) Unable to read file '{0}' encoding='{1}'.".format(
filename, encoding)) from e
try:
nb = reads(nbc)
except NotJSONError as e: # pragma: no cover
raise NotebookException(
"(1) Unable to read file '{0}' encoding='{1}'.".format(
filename, encoding)) from e
out = StringIO()
def flogging(*args, **kwargs):
if len(args) > 0:
out.write(" ".join(args))
if len(kwargs) > 0:
out.write(str(kwargs))
out.write("\n")
fLOG(*args, **kwargs)
try:
nb_runner = NotebookRunner(nb,
profile_dir,
working_dir,
fLOG=flogging,
filename=filename,
theNotebook=os.path.abspath(filename),
code_init=code_init,
log_level=log_level,
extended_args=extended_args,
kernel_name=kernel_name,
replacements=cached_rep,
kernel=True,
detailed_log=detailed_log,
startup_timeout=startup_timeout)
except NotebookKernelError: # pragma: no cover
# It fails. We try again once.
nb_runner = NotebookRunner(nb,
profile_dir,
working_dir,
fLOG=flogging,
filename=filename,
theNotebook=os.path.abspath(filename),
code_init=code_init,
log_level=log_level,
extended_args=extended_args,
kernel_name=kernel_name,
replacements=cached_rep,
kernel=True,
detailed_log=detailed_log,
startup_timeout=startup_timeout)
try:
stat = nb_runner.run_notebook(skip_exceptions=skip_exceptions,
additional_path=additional_path,
valid=valid,
clean_function=clean_function)
if outfilename is not None:
with open(outfilename, 'w', encoding=encoding) as f:
try:
s = writes(nb_runner.nb)
except NotebookException as e: # pragma: no cover
raise NotebookException(
"issue with notebook: '{}'".format(filename)) from e
if isinstance(s, bytes):
s = s.decode('utf8')
f.write(s)
finally:
nb_runner.shutdown_kernel()
return stat, out.getvalue()
def run_notebook(filename, profile_dir=None, working_dir=None, skip_exceptions=False,
outfilename=None, encoding="utf8", additional_path=None,
valid=None, clean_function=None, code_init=None,
fLOG=noLOG, kernel_name="python", log_level="30",
extended_args=None, cache_urls=None, replacements=None):
"""
run a notebook end to end, it uses module `runipy <https://github.com/paulgb/runipy/>`_
@param filename notebook filename
@param profile_dir profile directory
@param working_dir working directory
@param skip_exceptions skip exceptions
@param outfilename if not None, saves the output in this notebook
@param encoding encoding for the notebooks
@param additional_path additional paths for import
@param valid if not None, valid is a function which returns whether
or not the cell should be executed or not, if the function
returns None, the execution of the notebooks and skip the execution
of the other cells
@param clean_function function which cleans a cell's code before executing it (None for None)
@param code_init code to run before the execution of the notebook as if it was a cell
@param fLOG logging function
@param kernel_name kernel name, it can be None
@param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
@param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
see :ref:`l-ipython_notebook_args` for a full list
@param cache_urls list of urls to cache
@param replacements list of additional replacements, list of tuple
@return tuple (statistics, output)
@warning The function calls `basicConfig <https://docs.python.org/3.4/library/logging.html#logging.basicConfig>`_.
.. exref::
:title: Run a notebook end to end
::
from pyquickhelper.ipythonhelper import run_notebook
run_notebook("source.ipynb", working_dir="temp",
outfilename="modified.ipynb",
additional_path = [ "c:/temp/mymodule/src" ] )
The function adds the local variable ``theNotebook`` with
the absolute file name of the notebook.
The execution of a notebook might fail because it relies on remote data
specified by url. The function downloads the data first and stores it in
folder *working_dir* (must not be None). The url string is replaced by
the absolute path to the file.
.. versionchanged:: 1.3
Parameters *log_level*, *extended_args*, *kernel_name* were added.
.. versionchanged:: 1.4
Parameter *cache_urls* was added.
Function *valid* can return None and stops the execution of the notebook.
"""
cached_rep = _cache_url_to_file(cache_urls, working_dir, fLOG=fLOG)
if replacements is None:
replacements = cached_rep
elif cached_rep is not None:
cached_rep.update(replacements)
with open(filename, "r", encoding=encoding) as payload:
nb = reads(payload.read())
out = StringIO()
def flogging(*l, **p):
if len(l) > 0:
out.write(" ".join(l))
if len(p) > 0:
out.write(str(p))
out.write("\n")
fLOG(*l, **p)
nb_runner = NotebookRunner(
nb, profile_dir, working_dir, fLOG=flogging, filename=filename,
theNotebook=os.path.abspath(filename),
code_init=code_init, log_level=log_level,
extended_args=extended_args, kernel_name=kernel_name,
replacements=replacements, kernel=True)
stat = nb_runner.run_notebook(skip_exceptions=skip_exceptions, additional_path=additional_path,
valid=valid, clean_function=clean_function)
if outfilename is not None:
with open(outfilename, 'w', encoding=encoding) as f:
try:
s = writes(nb_runner.nb)
except NotebookException as e:
raise NotebookException(
"issue with notebook: " + filename) from e
if isinstance(s, bytes):
s = s.decode('utf8')
f.write(s)
nb_runner.shutdown_kernel()
return stat, out.getvalue()
