def copy_cells(from_notebook='', from_cells=(0, 0), to_notebook='', at_cell=0):
if '.ipynb' not in from_notebook:
from_notebook += '.ipynb'
if '.ipynb' not in to_notebook:
to_notebook += '.ipynb'
with open(from_notebook) as nb1_f:
nb1_raw = nb1_f.read()
with open(to_notebook) as nb2_f:
nb2_raw = nb2_f.read()
nb1 = nbformat.reads(nb1_raw, as_version=ipy_ver)
nb2 = nbformat.reads(nb2_raw, as_version=ipy_ver)
start_id, end_id = from_cells
copied_cells = nb1['worksheets'][0]['cells'][start_id:end_id]
active_cells = nb2['worksheets'][0]['cells']
nb2['worksheets'][0][
'cells'] = active_cells[:at_cell] + copied_cells + active_cells[
at_cell:]
nb2_modified_raw = nbformat.writes(nb2, as_version=ipy_ver)
with open(to_notebook, 'w') as nb2_f:
nb2_f.write(nb2_modified_raw)
def copy_cells(from_notebook='', from_cells=(0, 0), to_notebook='', at_cell=0):
if '.ipynb' not in from_notebook:
from_notebook += '.ipynb'
if '.ipynb' not in to_notebook:
to_notebook += '.ipynb'
with open(from_notebook) as nb1_f:
nb1_raw = nb1_f.read()
with open(to_notebook) as nb2_f:
nb2_raw = nb2_f.read()
nb1 = nbformat.reads(nb1_raw, as_version=ipy_ver)
nb2 = nbformat.reads(nb2_raw, as_version=ipy_ver)
start_id, end_id = from_cells
copied_cells = nb1['worksheets'][0]['cells'][start_id:end_id]
active_cells = nb2['worksheets'][0]['cells']
nb2['worksheets'][0]['cells'] = active_cells[:at_cell] + copied_cells + active_cells[at_cell:]
nb2_modified_raw = nbformat.writes(nb2, as_version=ipy_ver)
with open(to_notebook, 'w') as nb2_f:
nb2_f.write(nb2_modified_raw)
def testRunNotebooks(self):
notebook_dir = path.join('tests', 'input')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected_file = path.join('tests', 'expected', notebook_file)
notebook = ""
with open(notebook_path) as notebook_file:
notebook = notebook_file.read()
try:
# IPython 3
notebook = reads(notebook, 3)
except (TypeError, NBFormatError):
# IPython 2
notebook = reads(notebook, 'json')
runner = NotebookRunner(notebook, working_dir=notebook_dir)
runner.run_notebook(True)
expected = ""
with open(expected_file) as notebook_file:
expected = notebook_file.read()
try:
# IPython 3
expected = reads(expected, 3)
except (TypeError, NBFormatError):
# IPython 2
expected = reads(expected, 'json')
self.assert_notebooks_equal(expected, runner.nb)
def testRunNotebooks(self):
notebook_dir = path.join('tests', 'input')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected_file = path.join('tests', 'expected', notebook_file)
notebook = ""
with open(notebook_path) as notebook_file:
notebook = notebook_file.read()
try:
# IPython 3
notebook = reads(notebook, 3)
except (TypeError, NBFormatError):
# IPython 2
notebook = reads(notebook, 'json')
runner = NotebookRunner(notebook, working_dir=notebook_dir)
runner.run_notebook(True)
expected = ""
with open(expected_file) as notebook_file:
expected = notebook_file.read()
try:
# IPython 3
expected = reads(expected, 3)
except (TypeError, NBFormatError):
# IPython 2
expected = reads(expected, 'json')
self.assert_notebooks_equal(expected, runner.nb)
def diff_code(self, revision1, revision2):
self.keeper._checkout(revision1)
nb1 = current.reads(
open(
os.path.join(
self.keeper.work_dir,
self.keeper.project_name,
self.keeper.project_config['internal-path'],
self.notebook,
"notebook" + ".ipynb"),
"r").read(),
"ipynb")
self.keeper._checkout(revision2)
nb2 = current.reads(
open(
os.path.join(
self.keeper.work_dir,
self.keeper.project_name,
self.keeper.project_config['internal-path'],
self.notebook,
"notebook" + ".ipynb"),
"r").read(),
"ipynb")
result_nb = notebook_diff(nb1, nb2)
filename_placeholder = "{}-{}: {} and {}".format(
self.keeper.project_name, self.notebook, revision1, revision2)
app.add_notebook(result_nb, filename_placeholder)
print "This diff have number {}".format(len(app.notebooks) - 1)
def export_nb(nb_json, format):
import IPython.nbconvert as nbconvert
import IPython.nbformat as nbformat
nb = nbformat.reads(nb_json, nbformat.NO_CONVERT)
output = nbconvert.export_by_name(format, nb)
print(output[0])
def check_single_notebook(notebook_filename, timeout=500):
"""
Checks single notebook being given its full name
(executes cells one-by-one checking there are no exceptions, nothing more is guaranteed)
"""
# do we need validation of JSON?
with open(notebook_filename) as notebook_file:
notebook_content = nbformat.reads(notebook_file.read(), as_version=nbformat.current_nbformat)
os.chdir(os.path.dirname(notebook_filename))
_, client = manager.start_new_kernel()
for cell in notebook_content.cells:
if cell.cell_type == 'code':
message_id = client.execute(cell.source)
try:
message = client.get_shell_msg(message_id, timeout=timeout)
except Empty:
raise RuntimeError("Cell timed out: \n {}".format(cell.source))
if message['content']['status'] != 'ok':
traceback = message['content']['traceback']
description = "Cell failed: '{}'\n\n Traceback:\n{}".format(cell.source, '\n'.join(traceback))
raise RuntimeError(description)
client.stop_channels()
def check_single_notebook(notebook_filename, timeout=500):
"""
Checks single notebook being given its full name
(executes cells one-by-one checking there are no exceptions, nothing more is guaranteed)
"""
# do we need validation of JSON?
with open(notebook_filename) as notebook_file:
notebook_content = nbformat.reads(notebook_file.read(),
as_version=nbformat.current_nbformat)
os.chdir(os.path.dirname(notebook_filename))
_, client = manager.start_new_kernel()
for cell in notebook_content.cells:
if cell.cell_type == 'code':
message_id = client.execute(cell.source)
try:
message = client.get_shell_msg(message_id, timeout=timeout)
except Empty:
raise RuntimeError("Cell timed out: \n {}".format(
cell.source))
if message['content']['status'] != 'ok':
traceback = message['content']['traceback']
description = "Cell failed: '{}'\n\n Traceback:\n{}".format(
cell.source, '\n'.join(traceback))
raise RuntimeError(description)
client.stop_channels()
def _notebook_model(self, path, content=True):
"""Build a notebook model from shock node
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
self._get_notebook_list()
model = self._base_model(path)
if model['name'] not in self.nb_list:
raise web.HTTPError(400, u"Unavailable Notebook: %s" % (model['name']))
model['type'] = 'notebook'
model['last_modified'] = dateutil.parser.parse(self.nb_list[model['name']]['attributes']['last_modified'])
model['last_modified'].replace(tzinfo=tz.UTC)
model['created'] = dateutil.parser.parse(self.nb_list[model['name']]['attributes']['created'])
model['created'].replace(tzinfo=tz.UTC)
if content:
try:
node_path = "/%s?download" %self.nb_list[model['name']]['id']
node_data = self._get_shock_node(node_path, 'data')
except:
raise web.HTTPError(500, u'Notebook cannot be read')
try:
nb = nbformat.reads(node_data, as_version=4)
except Exception as e:
raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (model['name'], e))
self.mark_trusted_cells(nb, model['name'])
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def collect(self):
with self.fspath.open() as f:
payload = f.read()
self.notebook_folder = self.fspath.dirname
try:
# Ipython 3
self.nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
self.nb = reads(payload, 'json')
self.runner = NotebookRunner(self.nb)
cell_num = 1
for cell in self.runner.iter_code_cells():
yield IPyNbCell(self.name, self, cell_num, cell)
cell_num += 1
def collect(self):
with self.fspath.open() as f:
payload = f.read()
self.notebook_folder = self.fspath.dirname
try:
# Ipython 3
self.nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
self.nb = reads(payload, 'json')
self.runner = NotebookRunner(self.nb)
cell_num = 1
for cell in self.runner.iter_code_cells():
yield IPyNbCell(self.name, self, cell_num, cell)
cell_num += 1
def create_post(self, path, **kw):
"""Create a new post."""
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
content = kw.pop('content', None)
onefile = kw.pop('onefile', False)
kernel = kw.pop('ipython_kernel', None)
# is_page is not needed to create the file
kw.pop('is_page', False)
metadata = {}
metadata.update(self.default_metadata)
metadata.update(kw)
makedirs(os.path.dirname(path))
if content.startswith("{"):
# imported .ipynb file, guaranteed to start with "{" because it’s JSON.
nb = nbformat.reads(content, current_nbformat)
else:
if IPython.version_info[0] >= 3:
nb = nbformat.v4.new_notebook()
nb["cells"] = [nbformat.v4.new_markdown_cell(content)]
else:
nb = nbformat.new_notebook()
nb["worksheets"] = [nbformat.new_worksheet(cells=[nbformat.new_text_cell('markdown', [content])])]
if kernelspec is not None:
if kernel is None:
kernel = self.default_kernel
self.logger.notice('No kernel specified, assuming "{0}".'.format(kernel))
IPYNB_KERNELS = {}
ksm = kernelspec.KernelSpecManager()
for k in ksm.find_kernel_specs():
IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict()
IPYNB_KERNELS[k]['name'] = k
del IPYNB_KERNELS[k]['argv']
if kernel not in IPYNB_KERNELS:
self.logger.error('Unknown kernel "{0}". Maybe you mispelled it?'.format(kernel))
self.logger.info("Available kernels: {0}".format(", ".join(sorted(IPYNB_KERNELS))))
raise Exception('Unknown kernel "{0}"'.format(kernel))
nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel]
else:
# Older IPython versions don’t need kernelspecs.
pass
if onefile:
nb["metadata"]["nikola"] = metadata
with io.open(path, "w+", encoding="utf8") as fd:
if IPython.version_info[0] >= 3:
nbformat.write(nb, fd, 4)
else:
nbformat.write(nb, fd, 'ipynb')
def finish_notebook(self, json_notebook, download_url, provider_url=None,
provider_icon=None, provider_label=None, msg=None,
breadcrumbs=None, public=False, format=None, request=None):
"""render a notebook from its JSON body.
download_url is required, provider_url is not.
msg is extra information for the log message when rendering fails.
"""
if msg is None:
msg = download_url
try:
nb = reads(json_notebook, current_nbformat)
except ValueError:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, "Error reading JSON notebook")
try:
app_log.debug("Requesting render of %s", download_url)
with self.time_block("Rendered %s" % download_url):
app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url)
nbhtml, config = yield self.pool.submit(render_notebook,
self.formats[format], nb, download_url,
config=self.config,
)
except NbFormatError as e:
app_log.error("Invalid notebook %s: %s", msg, e)
raise web.HTTPError(400, str(e))
except Exception as e:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, str(e))
else:
app_log.debug("Finished render of %s", download_url)
html = self.render_template(
"formats/%s.html" % format,
body=nbhtml,
nb=nb,
download_url=download_url,
provider_url=provider_url,
provider_label=provider_label,
provider_icon=provider_icon,
format=self.format,
default_format=self.default_format,
format_prefix=format_prefix,
formats=dict(self.filter_formats(nb, json_notebook)),
format_base=self.request.uri.replace(self.format_prefix, ""),
date=datetime.utcnow().strftime(date_fmt),
breadcrumbs=breadcrumbs,
**config)
yield self.cache_and_finish(html)
# Index notebook
self.index.index_notebook(download_url, nb, public)
def finish_notebook(self, json_notebook, download_url, provider_url=None,
provider_icon=None, provider_label=None, msg=None,
breadcrumbs=None, public=False, format=None, request=None):
"""render a notebook from its JSON body.
download_url is required, provider_url is not.
msg is extra information for the log message when rendering fails.
"""
if msg is None:
msg = download_url
try:
nb = reads(json_notebook, current_nbformat)
except ValueError:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, "Error reading JSON notebook")
try:
app_log.debug("Requesting render of %s", download_url)
with self.time_block("Rendered %s" % download_url):
app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url)
nbhtml, config = yield self.pool.submit(render_notebook,
self.formats[format], nb, download_url,
config=self.config,
)
except NbFormatError as e:
app_log.error("Invalid notebook %s: %s", msg, e)
raise web.HTTPError(400, str(e))
except Exception as e:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, str(e))
else:
app_log.debug("Finished render of %s", download_url)
html = self.render_template(
"formats/%s.html" % format,
body=nbhtml,
nb=nb,
download_url=download_url,
provider_url=provider_url,
provider_label=provider_label,
provider_icon=provider_icon,
format=self.format,
default_format=self.default_format,
format_prefix=format_prefix,
formats=dict(self.filter_formats(nb, json_notebook)),
format_base=self.request.uri.replace(self.format_prefix, ""),
date=datetime.utcnow().strftime(date_fmt),
breadcrumbs=breadcrumbs,
**config)
yield self.cache_and_finish(html)
# Index notebook
self.index.index_notebook(download_url, nb, public)
def testUseCLI(self):
notebook_dir = path.join('tests', 'expected')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected = ""
with open(notebook_path) as notebook_file:
expected = notebook_file.read()
try:
# IPython 3
expected = reads(expected, 3)
except (TypeError, NBFormatError):
# IPython 2
expected = reads(expected, 'json')
exit_code = 1
argv = sys.argv
stdout = sys.stdout
stderr = sys.stderr
try:
with open(devnull, "w") as devnull_filehandle:
sys.stdout = sys.stderr = devnull_filehandle
sys.argv = [
"runipy",
"-o", notebook_path,
"--html", notebook_path.replace(".ipynb", ".html")
]
main()
except SystemExit as e:
exit_code = e.code
finally:
sys.argv = argv
sys.stdout = stdout
sys.stderr = stderr
notebook = ""
with open(notebook_path) as notebook_file:
notebook = notebook_file.read()
try:
# IPython 3
notebook = reads(notebook, 3)
except (TypeError, NBFormatError):
# IPython 2
notebook = reads(notebook, 'json')
self.assert_notebooks_equal(expected, notebook)
def testUseCLI(self):
notebook_dir = path.join('tests', 'expected')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected = ""
with open(notebook_path) as notebook_file:
expected = notebook_file.read()
try:
# IPython 3
expected = reads(expected, 3)
except (TypeError, NBFormatError):
# IPython 2
expected = reads(expected, 'json')
exit_code = 1
argv = sys.argv
stdout = sys.stdout
stderr = sys.stderr
try:
with open(devnull, "w") as devnull_filehandle:
sys.stdout = sys.stderr = devnull_filehandle
sys.argv = [
"runipy", "-o", notebook_path, "--html",
notebook_path.replace(".ipynb", ".html")
]
main()
except SystemExit as e:
exit_code = e.code
finally:
sys.argv = argv
sys.stdout = stdout
sys.stderr = stderr
notebook = ""
with open(notebook_path) as notebook_file:
notebook = notebook_file.read()
try:
# IPython 3
notebook = reads(notebook, 3)
except (TypeError, NBFormatError):
# IPython 2
notebook = reads(notebook, 'json')
self.assert_notebooks_equal(expected, notebook)
def _read_notebook(self, os_path, as_version=4):
"""Read a notebook from an os path."""
with self.open(os_path, "r", encoding="utf-8") as f:
try:
if ftdetect(os_path) == "notebook":
return nbformat.read(f, as_version=as_version)
elif ftdetect(os_path) == "markdown":
nbjson = convert(os_path, informat="markdown", outformat="notebook")
return nbformat.reads(nbjson, as_version=as_version)
except Exception as e:
raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e))
def _read_notebook(self, os_path, as_version=4):
"""Read a notebook from an os path."""
with self.open(os_path, 'r', encoding='utf-8') as f:
try:
if ftdetect(os_path) == 'notebook':
return nbformat.read(f, as_version=as_version)
elif ftdetect(os_path) == 'markdown':
nbjson = convert(os_path,
informat='markdown',
outformat='notebook')
return nbformat.reads(nbjson, as_version=as_version)
except Exception as e:
raise web.HTTPError(
400,
u"Unreadable Notebook: %s %r" % (os_path, e),
)
def render(self):
try:
with open(self.file_path, 'r') as file_pointer:
notebook = nbformat.reads(file_pointer.read(), as_version=4)
except ValueError:
raise exceptions.InvalidFormat('Could not read ipython notebook file.')
exporter = HTMLExporter(config=Config({
'HTMLExporter': {
'template_file': 'basic',
},
'CSSHtmlHeaderTransformer': {
'enabled': False,
},
}))
(body, _) = exporter.from_notebook_node(notebook)
return self.TEMPLATE.render(base=self.assets_url, body=body)
def get_notebook(self, name, path='', content=True, **kwargs):
""" Takes a path and name for a notebook and returns its model
Parameters
----------
name : str
the name of the notebook
path : str
the URL path that describes the relative path for
the notebook
Returns
-------
model : dict
the notebook model. If contents=True, returns the 'contents'
dict in the model as well.
"""
path = path.strip('/')
if not self.notebook_exists(name=name, path=path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
gist = self._get_gist(name, path)
# Create the notebook model.
model = {}
model['name'] = name
model['path'] = path
model['last_modified'] = gist.updated_at
model['created'] = gist.created_at
model['type'] = 'notebook'
model['format'] = None
model['writable'] = True
model['mimetype'] = None
if content:
model['format'] = 'json'
notebook_content = gist.notebook_content
try:
nb = nbformat.reads(notebook_content, as_version=4)
except Exception as e:
raise web.HTTPError(
400, u"Unreadable Notebook: %s %s %s" % (path, name, e))
self.mark_trusted_cells(nb, self.fullpath(path, name))
# add gist id if public.
if gist.public:
nb['metadata']['gist_id'] = gist.id
model['content'] = nb
return model
def get_notebook(self, name, path='', content=True, **kwargs):
""" Takes a path and name for a notebook and returns its model
Parameters
----------
name : str
the name of the notebook
path : str
the URL path that describes the relative path for
the notebook
Returns
-------
model : dict
the notebook model. If contents=True, returns the 'contents'
dict in the model as well.
"""
path = path.strip('/')
if not self.notebook_exists(name=name, path=path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
gist = self._get_gist(name, path)
# Create the notebook model.
model ={}
model['name'] = name
model['path'] = path
model['last_modified'] = gist.updated_at
model['created'] = gist.created_at
model['type'] = 'notebook'
model['format'] = None
model['writable'] = True
model['mimetype'] = None
if content:
model['format'] = 'json'
notebook_content = gist.notebook_content
try:
nb = nbformat.reads(notebook_content, as_version=4)
except Exception as e:
raise web.HTTPError(400, u"Unreadable Notebook: %s %s %s" % (path, name, e))
self.mark_trusted_cells(nb, self.fullpath(path, name))
# add gist id if public.
if gist.public:
nb['metadata']['gist_id'] = gist.id
model['content'] = nb
return model
def render(self):
try:
with open(self.file_path, 'r') as file_pointer:
notebook = nbformat.reads(file_pointer.read(), as_version=4)
except ValueError:
raise exceptions.InvalidFormat(
'Could not read ipython notebook file.')
exporter = HTMLExporter(config=Config({
'HTMLExporter': {
'template_file': 'basic',
},
'CSSHtmlHeaderTransformer': {
'enabled': False,
},
}))
(body, _) = exporter.from_notebook_node(notebook)
return self.TEMPLATE.render(base=self.assets_url, body=body)
def test_roundtrip():
"""Run nbconvert using our custom markdown template to recover
original markdown from a notebook.
"""
# create a notebook from the markdown
mr = notedown.MarkdownReader()
roundtrip_notebook = mr.to_notebook(roundtrip_markdown)
# write the notebook into json
notebook_json = nbformat.writes(roundtrip_notebook)
# write the json back into notebook
notebook = nbformat.reads(notebook_json, as_version=4)
# convert notebook to markdown
mw = notedown.MarkdownWriter(template_file='notedown/templates/markdown.tpl', strip_outputs=True)
markdown = mw.writes(notebook)
nt.assert_multi_line_equal(roundtrip_markdown, markdown)
def test_roundtrip():
"""Run nbconvert using our custom markdown template to recover
original markdown from a notebook.
"""
# create a notebook from the markdown
mr = notedown.MarkdownReader()
roundtrip_notebook = mr.to_notebook(roundtrip_markdown)
# write the notebook into json
notebook_json = nbformat.writes(roundtrip_notebook)
# write the json back into notebook
notebook = nbformat.reads(notebook_json, as_version=4)
# convert notebook to markdown
mw = notedown.MarkdownWriter(
template_file='notedown/templates/markdown.tpl', strip_outputs=True)
markdown = mw.writes(notebook)
nt.assert_multi_line_equal(roundtrip_markdown, markdown)
def collect(self):
with self.fspath.open() as f:
# self.nb = reads(f.read(), 'json')
self.nb = reads(f.read(), 4)
# Start the cell count
cell_num = 0
# Worksheets are NOT used anymore::
# Currently there is only 1 worksheet (it seems in newer versions
# of IPython, they are going to get rid of this option)
# For every worksheet, read every cell associated to it
for cell in self.nb.cells:
# Skip the cells that have text, headings or related stuff
# Only test code cells
if cell.cell_type == 'code':
# If the code is a notebook magic cell, do not run
# i.e. cell code starts with '%%'
# Also ignore the cells that start with the
# comment string PYTEST_VALIDATE_IGNORE_OUTPUT
# NOTE: This actually skips execution, which probably isn't what we want!
# It is typically helpful to execute the cell (to make sure that at
# least the code doesn't fail) but then discard the result.
if not (cell.source.startswith('%%') or
cell.source.startswith(r'# PYTEST_VALIDATE_IGNORE_OUTPUT') or
cell.source.startswith(r'#PYTEST_VALIDATE_IGNORE_OUTPUT')):
yield IPyNbCell(self.name, self, cell_num, cell)
else:
# Skipped cells will not be counted
continue
# Update 'code' cell count
cell_num += 1
def collect(self):
with self.fspath.open() as f:
# self.nb = reads(f.read(), 'json')
self.nb = reads(f.read(), 4)
# Start the cell count
cell_num = 0
# Worksheets are NOT used anymore::
# Currently there is only 1 worksheet (it seems in newer versions
# of IPython, they are going to get rid of this option)
# For every worksheet, read every cell associated to it
for cell in self.nb.cells:
# Skip the cells that have text, headings or related stuff
# Only test code cells
if cell.cell_type == 'code':
# If the code is a notebook magic cell, do not run
# i.e. cell code starts with '%%'
# Also ignore the cells that start with the
# comment string PYTEST_VALIDATE_IGNORE_OUTPUT
# NOTE: This actually skips execution, which probably isn't what we want!
# It is typically helpful to execute the cell (to make sure that at
# least the code doesn't fail) but then discard the result.
if not (cell.source.startswith('%%') or
cell.source.startswith(r'# PYTEST_VALIDATE_IGNORE_OUTPUT') or
cell.source.startswith(r'#PYTEST_VALIDATE_IGNORE_OUTPUT')):
yield IPyNbCell(self.name, self, cell_num, cell)
else:
# Skipped cells will not be counted
continue
# Update 'code' cell count
cell_num += 1
def export_nb(nb_json, format):
import IPython.nbconvert as nbconvert
import IPython.nbformat as nbformat
nb = nbformat.reads(nb_json, nbformat.NO_CONVERT)
output = nbconvert.export_by_name(format, nb)
print(output[0])
def readjson(json_str):
data = current.reads(json_str, 4)
return data
def reads_base64(nb, as_version=NBFORMAT_VERSION):
"""
Read a notebook from base64.
"""
return reads(b64decode(nb).decode('utf-8'), as_version=as_version)
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
reply = shell.get_msg(timeout=20000)['content']
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:")
print(cell.input)
print('-----')
print("raised:")
print('\n'.join(reply['traceback']))
cells += 1
sys.stdout.write('.')
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
if __name__ == '__main__':
for ipynb in sys.argv[1:]:
print("running %s" % ipynb)
with open(ipynb) as f:
nb = reads(f.read(), 'json')
run_notebook(nb)
def main():
log_format = "%(asctime)s %(levelname)s: %(message)s"
log_datefmt = "%m/%d/%Y %I:%M:%S %p"
parser = argparse.ArgumentParser()
parser.add_argument(
"--version", "-v", action="version", version=runipy.__version__, help="print version information"
)
parser.add_argument("input_file", nargs="?", help=".ipynb file to run (or stdin)")
parser.add_argument("output_file", nargs="?", help=".ipynb file to save cell output to")
parser.add_argument("--quiet", "-q", action="store_true", help="don't print anything unless things go wrong")
parser.add_argument(
"--overwrite", "-o", action="store_true", help="write notebook output back to original notebook"
)
format_grp = parser.add_mutually_exclusive_group()
format_grp.add_argument("--html", nargs="?", default=False, help="output an HTML snapshot of the notebook")
format_grp.add_argument("--pdf", nargs="?", default=False, help="output a PDF snapshot of the notebook")
parser.add_argument("--template", nargs="?", default=False, help="template to use for HTML output")
parser.add_argument("--pylab", action="store_true", help="start notebook with pylab enabled")
parser.add_argument("--matplotlib", action="store_true", help="start notebook with matplotlib inlined")
parser.add_argument(
"--skip-exceptions",
"-s",
action="store_true",
help="if an exception occurs in a cell," + " continue running the subsequent cells",
)
parser.add_argument("--stdout", action="store_true", help="print notebook to stdout (or use - as output_file")
parser.add_argument("--stdin", action="store_true", help="read notebook from stdin (or use - as input_file)")
parser.add_argument(
"--no-chdir", action="store_true", help="do not change directory to notebook's at kernel startup"
)
parser.add_argument("--profile-dir", help="set the profile location directly")
args = parser.parse_args()
if args.overwrite:
if args.output_file is not None:
print("Error: output_filename must not be provided if " "--overwrite (-o) given", file=stderr)
exit(1)
else:
args.output_file = args.input_file
if not args.quiet:
logging.basicConfig(level=logging.INFO, format=log_format, datefmt=log_datefmt)
working_dir = None
payload_source = ""
payload = ""
if args.input_file == "-" or args.stdin: # force stdin
payload_source = stdin.name
payload = stdin.read()
elif not args.input_file and stdin.isatty(): # no force, empty stdin
parser.print_help()
exit()
elif not args.input_file: # no file -> default stdin
payload_source = stdin.name
payload = stdin.read()
else: # must have specified normal input_file
with open(args.input_file) as input_file:
payload_source = input_file.name
payload = input_file.read()
working_dir = os.path.dirname(args.input_file)
if args.no_chdir:
working_dir = None
if args.profile_dir:
profile_dir = os.path.expanduser(args.profile_dir)
else:
profile_dir = None
logging.info("Reading notebook %s", payload_source)
try:
# Ipython 3
nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
nb = reads(payload, "json")
nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir, working_dir)
exit_status = 0
try:
nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
except NotebookError:
exit_status = 1
if args.output_file and args.output_file != "-":
logging.info("Saving to %s", args.output_file)
with open(args.output_file, "w") as output_filehandle:
try:
# Ipython 3
write(nb_runner.nb, output_filehandle, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, output_filehandle, "json")
if args.stdout or args.output_file == "-":
try:
# Ipython 3
write(nb_runner.nb, stdout, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, stdout, "json")
print()
if args.html is not False:
if args.html is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith(".ipynb"):
args.html = args.input_file[:-6] + ".html"
else:
args.html = args.input_file + ".html"
if args.template is False:
exporter = HTMLExporter()
else:
exporter = HTMLExporter(
config=Config({"HTMLExporter": {"template_file": args.template, "template_path": [".", "/"]}})
)
logging.info("Saving HTML snapshot to %s" % args.html)
output, resources = exporter.from_notebook_node(convert(nb_runner.nb, current_nbformat))
codecs.open(args.html, "w", encoding="utf-8").write(output)
elif args.pdf is not False:
if args.pdf is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith(".ipynb"):
args.pdf = args.input_file[:-6] + ".pdf"
else:
args.pdf = args.input_file + ".pdf"
if args.template is False:
exporter = PDFExporter()
else:
exporter = PDFExporter(
config=Config({"PDFExporter": {"template_file": args.template, "template_path": [".", "/"]}})
)
logging.info("Saving PDF snapshot to %s" % args.pdf)
output, resources = exporter.from_notebook_node(convert(nb_runner.nb, current_nbformat))
writer = FilesWriter()
writer.build_directory = os.path.dirname(args.pdf)
writer.write(output, resources, os.path.splitext(os.path.basename(args.pdf))[0])
nb_runner.shutdown_kernel()
if exit_status != 0:
logging.warning("Exiting with nonzero exit status")
exit(exit_status)
def main():
log_format = '%(asctime)s %(levelname)s: %(message)s'
log_datefmt = '%m/%d/%Y %I:%M:%S %p'
parser = argparse.ArgumentParser()
parser.add_argument(
'--version', '-v', action='version',
version=runipy.__version__,
help='print version information'
)
parser.add_argument(
'input_file', nargs='?',
help='.ipynb file to run (or stdin)'
)
parser.add_argument(
'output_file', nargs='?',
help='.ipynb file to save cell output to'
)
parser.add_argument(
'--quiet', '-q', action='store_true',
help='don\'t print anything unless things go wrong'
)
parser.add_argument(
'--overwrite', '-o', action='store_true',
help='write notebook output back to original notebook'
)
parser.add_argument(
'--html', nargs='?', default=False,
help='output an HTML snapshot of the notebook'
)
parser.add_argument(
'--template', nargs='?', default=False,
help='template to use for HTML output'
)
parser.add_argument(
'--pylab', action='store_true',
help='start notebook with pylab enabled'
)
parser.add_argument(
'--matplotlib', action='store_true',
help='start notebook with matplotlib inlined'
)
parser.add_argument(
'--skip-exceptions', '-s', action='store_true',
help='if an exception occurs in a cell,' +
' continue running the subsequent cells'
)
parser.add_argument(
'--stdout', action='store_true',
help='print notebook to stdout (or use - as output_file'
)
parser.add_argument(
'--stdin', action='store_true',
help='read notebook from stdin (or use - as input_file)'
)
parser.add_argument(
'--no-chdir', action='store_true',
help="do not change directory to notebook's at kernel startup"
)
parser.add_argument(
'--profile-dir',
help="set the profile location directly"
)
args = parser.parse_args()
if args.overwrite:
if args.output_file is not None:
print('Error: output_filename must not be provided if '
'--overwrite (-o) given', file=stderr)
exit(1)
else:
args.output_file = args.input_file
if not args.quiet:
logging.basicConfig(
level=logging.INFO, format=log_format, datefmt=log_datefmt
)
working_dir = None
payload_source = ""
payload = ""
if args.input_file == '-' or args.stdin: # force stdin
payload_source = stdin.name
payload = stdin.read()
elif not args.input_file and stdin.isatty(): # no force, empty stdin
parser.print_help()
exit()
elif not args.input_file: # no file -> default stdin
payload_source = stdin.name
payload = stdin.read()
else: # must have specified normal input_file
with open(args.input_file) as input_file:
payload_source = input_file.name
payload = input_file.read()
working_dir = os.path.dirname(args.input_file)
if args.no_chdir:
working_dir = None
if args.profile_dir:
profile_dir = os.path.expanduser(args.profile_dir)
else:
profile_dir = None
logging.info('Reading notebook %s', payload_source)
try:
# Ipython 3
nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
nb = reads(payload, 'json')
nb_runner = NotebookRunner(
nb, args.pylab, args.matplotlib, profile_dir, working_dir
)
exit_status = 0
try:
nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
except NotebookError:
exit_status = 1
if args.output_file and args.output_file != '-':
logging.info('Saving to %s', args.output_file)
with open(args.output_file, 'w') as output_filehandle:
try:
# Ipython 3
write(nb_runner.nb, output_filehandle, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, output_filehandle, 'json')
if args.stdout or args.output_file == '-':
try:
# Ipython 3
write(nb_runner.nb, stdout, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, stdout, 'json')
print()
if args.html is not False:
if args.html is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith('.ipynb'):
args.html = args.input_file[:-6] + '.html'
else:
args.html = args.input_file + '.html'
if args.template is False:
exporter = HTMLExporter()
else:
exporter = HTMLExporter(
config=Config({
'HTMLExporter': {
'template_file': args.template,
'template_path': ['.', '/']
}
})
)
logging.info('Saving HTML snapshot to %s' % args.html)
output, resources = exporter.from_notebook_node(nb_runner.nb)
codecs.open(args.html, 'w', encoding='utf-8').write(output)
nb_runner.shutdown_kernel()
if exit_status != 0:
logging.warning('Exiting with nonzero exit status')
exit(exit_status)
def main():
log_format = '%(asctime)s %(levelname)s: %(message)s'
log_datefmt = '%m/%d/%Y %I:%M:%S %p'
parser = argparse.ArgumentParser()
parser.add_argument('--version',
'-v',
action='version',
version=runipy.__version__,
help='print version information')
parser.add_argument('input_file',
nargs='?',
help='.ipynb file to run (or stdin)')
parser.add_argument('output_file',
nargs='?',
help='.ipynb file to save cell output to')
parser.add_argument('--quiet',
'-q',
action='store_true',
help='don\'t print anything unless things go wrong')
parser.add_argument('--overwrite',
'-o',
action='store_true',
help='write notebook output back to original notebook')
parser.add_argument('--html',
nargs='?',
default=False,
help='output an HTML snapshot of the notebook')
parser.add_argument('--template',
nargs='?',
default=False,
help='template to use for HTML output')
parser.add_argument('--pylab',
action='store_true',
help='start notebook with pylab enabled')
parser.add_argument('--matplotlib',
action='store_true',
help='start notebook with matplotlib inlined')
parser.add_argument('--skip-exceptions',
'-s',
action='store_true',
help='if an exception occurs in a cell,' +
' continue running the subsequent cells')
parser.add_argument(
'--stdout',
action='store_true',
help='print notebook to stdout (or use - as output_file')
parser.add_argument(
'--stdin',
action='store_true',
help='read notebook from stdin (or use - as input_file)')
parser.add_argument(
'--no-chdir',
action='store_true',
help="do not change directory to notebook's at kernel startup")
parser.add_argument('--profile-dir',
help="set the profile location directly")
args = parser.parse_args()
if args.overwrite:
if args.output_file is not None:
print(
'Error: output_filename must not be provided if '
'--overwrite (-o) given',
file=stderr)
exit(1)
else:
args.output_file = args.input_file
if not args.quiet:
logging.basicConfig(level=logging.INFO,
format=log_format,
datefmt=log_datefmt)
working_dir = None
payload_source = ""
payload = ""
if args.input_file == '-' or args.stdin: # force stdin
payload_source = stdin.name
payload = stdin.read()
elif not args.input_file and stdin.isatty(): # no force, empty stdin
parser.print_help()
exit()
elif not args.input_file: # no file -> default stdin
payload_source = stdin.name
payload = stdin.read()
else: # must have specified normal input_file
with open(args.input_file) as input_file:
payload_source = input_file.name
payload = input_file.read()
working_dir = os.path.dirname(args.input_file)
if args.no_chdir:
working_dir = None
if args.profile_dir:
profile_dir = os.path.expanduser(args.profile_dir)
else:
profile_dir = None
logging.info('Reading notebook %s', payload_source)
try:
# Ipython 3
nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
nb = reads(payload, 'json')
nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir,
working_dir)
exit_status = 0
try:
nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
except NotebookError:
exit_status = 1
if args.output_file and args.output_file != '-':
logging.info('Saving to %s', args.output_file)
with open(args.output_file, 'w') as output_filehandle:
try:
# Ipython 3
write(nb_runner.nb, output_filehandle, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, output_filehandle, 'json')
if args.stdout or args.output_file == '-':
try:
# Ipython 3
write(nb_runner.nb, stdout, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, stdout, 'json')
print()
if args.html is not False:
if args.html is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith('.ipynb'):
args.html = args.input_file[:-6] + '.html'
else:
args.html = args.input_file + '.html'
if args.template is False:
exporter = HTMLExporter()
else:
exporter = HTMLExporter(config=Config({
'HTMLExporter': {
'template_file': args.template,
'template_path': ['.', '/']
}
}))
logging.info('Saving HTML snapshot to %s' % args.html)
output, resources = exporter.from_notebook_node(
convert(nb_runner.nb, current_nbformat))
codecs.open(args.html, 'w', encoding='utf-8').write(output)
nb_runner.shutdown_kernel()
if exit_status != 0:
logging.warning('Exiting with nonzero exit status')
exit(exit_status)
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
reply = shell.get_msg(timeout=20000)['content']
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:")
print(cell.input)
print('-----')
print("raised:")
print('\n'.join(reply['traceback']))
cells += 1
sys.stdout.write('.')
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
if __name__ == '__main__':
for ipynb in sys.argv[1:]:
print("running %s" % ipynb)
with open(ipynb) as f:
nb = reads(f.read(), 'json')
run_notebook(nb)
def create_post(self, path, **kw):
"""Create a new post."""
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'],
'build this site (compile ipynb)')
content = kw.pop('content', None)
onefile = kw.pop('onefile', False)
kernel = kw.pop('ipython_kernel', None)
# is_page is not needed to create the file
kw.pop('is_page', False)
metadata = {}
metadata.update(self.default_metadata)
metadata.update(kw)
makedirs(os.path.dirname(path))
if content.startswith("{"):
# imported .ipynb file, guaranteed to start with "{" because it’s JSON.
nb = nbformat.reads(content, current_nbformat)
else:
if IPython.version_info[0] >= 3:
nb = nbformat.v4.new_notebook()
nb["cells"] = [nbformat.v4.new_markdown_cell(content)]
else:
nb = nbformat.new_notebook()
nb["worksheets"] = [
nbformat.new_worksheet(
cells=[nbformat.new_text_cell('markdown', [content])])
]
if kernelspec is not None:
if kernel is None:
kernel = self.default_kernel
self.logger.notice(
'No kernel specified, assuming "{0}".'.format(kernel))
IPYNB_KERNELS = {}
ksm = kernelspec.KernelSpecManager()
for k in ksm.find_kernel_specs():
IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict()
IPYNB_KERNELS[k]['name'] = k
del IPYNB_KERNELS[k]['argv']
if kernel not in IPYNB_KERNELS:
self.logger.error(
'Unknown kernel "{0}". Maybe you mispelled it?'.format(
kernel))
self.logger.info("Available kernels: {0}".format(", ".join(
sorted(IPYNB_KERNELS))))
raise Exception('Unknown kernel "{0}"'.format(kernel))
nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel]
else:
# Older IPython versions don’t need kernelspecs.
pass
if onefile:
nb["metadata"]["nikola"] = metadata
with io.open(path, "w+", encoding="utf8") as fd:
if IPython.version_info[0] >= 3:
nbformat.write(nb, fd, 4)
else:
nbformat.write(nb, fd, 'ipynb')
def reads_base64(nb, as_version=NBFORMAT_VERSION):
"""
Read a notebook from base64.
"""
return reads(b64decode(nb).decode('utf-8'), as_version=as_version)
